From 036cc67acde119b09d2c406450bf362d83a2a2d5 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Thu, 2 Apr 2026 12:51:01 +0200 Subject: [PATCH 01/20] lstm + egu conference --- pyproject.toml | 4 +- scripts/compare_runs.py | 18 +- scripts/create_ensemble.py | 573 ++++++++++++++++++ scripts/explain_tft.py | 62 +- st_forecast/custom_models/ExoLSTM.py | 4 +- st_forecast/custom_models/ExoMamba.py | 4 +- st_forecast/custom_models/FANMixer.py | 4 +- st_forecast/custom_models/lstm/module.py | 4 +- .../custom_models/lstm_enc_dec/module.py | 4 +- .../custom_models/lstm_enc_mlp/module.py | 4 +- .../custom_models/mamba_enc_dec/module.py | 4 +- .../custom_models/unified_temporal/module.py | 6 +- st_forecast/evaluate_model.py | 14 +- st_forecast/model_helper.py | 19 + st_forecast/pipeline/artifacts.py | 1 + st_forecast/pipeline/inference.py | 8 +- st_forecast/pipeline/training.py | 4 + st_forecast/visualization/__init__.py | 8 + st_forecast/visualization/_helpers.py | 5 +- st_forecast/visualization/_styles.py | 25 + st_forecast/visualization/compare_runs.py | 248 +++++--- tests/test_inference.py | 8 +- uv.lock | 275 ++------- 23 files changed, 966 insertions(+), 340 deletions(-) create mode 100644 scripts/create_ensemble.py diff --git a/pyproject.toml b/pyproject.toml index 438dbbb..4f94605 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.11.9" dependencies = [ "click>=8.0", - "darts==0.35.0", + "darts==0.43.0", "joblib>=1.5.2", "lightning==2.5.1", "mambapy==1.2.0", @@ -16,9 +16,11 @@ dependencies = [ "pandas>=2.3.3", "pe-oudin==0.3", "psutil==5.9.8", + "pyarrow>=23.0.1", "scikit-learn>=1.7.2", "seaborn>=0.13.2", "suntime==1.3.2", + "tensorboardx>=2.6.4", "tqdm>=4.67.1", ] diff --git a/scripts/compare_runs.py b/scripts/compare_runs.py index 6ccdd96..e393260 100644 --- a/scripts/compare_runs.py +++ b/scripts/compare_runs.py @@ -15,37 +15,43 @@ RUN_FOLDERS: list[Path] = [ Path("./runs/persistence_kgz"), + Path("./runs/egu/kgz_lstm"), Path("./runs/egu/tsmixer_kgz"), Path("./runs/egu/tide_kgz"), Path("./runs/egu/tft_kgz"), - Path("./runs/egu/enc_dec_kgz"), + Path("./runs/ensemble_kgz"), # Add more runs as needed ] -RUN_FOLDERS: list[Path] = [ +"""RUN_FOLDERS: list[Path] = [ Path("./runs/persistence_taj"), + Path("./runs/egu/taj_lstm"), Path("./runs/egu/tsmixer_taj"), Path("./runs/egu/tide_taj"), Path("./runs/egu/tft_taj"), - -] + Path("./runs/ensemble_taj"), +]""" RUN_LABELS: list[str] = [ "Persistence", + "LSTM", "TSMixer", "TiDE", "TFT", - "LSTMEncMLP", + "Ensemble", ] RUN_LABELS: list[str] = [ "Persistence", + "LSTM", "TSMixer", "TiDE", "TFT", + "Ensemble", + ] -OUTPUT_DIR: Path = Path("comparison_output/taj") +OUTPUT_DIR: Path = Path("comparison_output/kgz") SPLIT: str = "test" # "train", "val", or "test" diff --git a/scripts/create_ensemble.py b/scripts/create_ensemble.py new file mode 100644 index 0000000..64c1c8c --- /dev/null +++ b/scripts/create_ensemble.py @@ -0,0 +1,573 @@ +"""Create ensemble predictions by mixing quantile CDFs from multiple models. + +Loads predictions from ensemble members, builds a mixture distribution at each +(date, code, forecast_step) triplet, extracts quantile predictions, evaluates +metrics, and generates visualizations including epistemic uncertainty plots. + +Usage: + uv run python scripts/create_ensemble.py +""" + +from __future__ import annotations + +import logging +from functools import reduce +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.evaluation.evaluator import EvaluationConfig, evaluate +from st_forecast.pipeline.artifacts import RunConfig, load_run_artifacts +from st_forecast.pipeline.inference import save_predictions +from st_forecast.visualization import ( + VisualizationConfig, + generate_visualizations, + select_representative_basins, +) +from st_forecast.visualization._styles import TOL + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s" +) +logger = logging.getLogger(__name__) + +# ============================================================================= +# CONFIGURATION - Edit these variables +# ============================================================================= + +ENSEMBLE_MEMBERS: list[Path] = [ + Path("./runs/egu/tsmixer_kgz"), + Path("./runs/egu/tide_kgz"), + Path("./runs/egu/tft_kgz"), + Path("./runs/egu/kgz_lstm"), +] + +OUTPUT_DIR: Path = Path("./runs/ensemble_kgz") + +SPLIT: str = "test" + +QUANTILE_LEVELS: list[float] = [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95] +QUANTILE_COLS: list[str] = ["Q5", "Q10", "Q25", "Q50", "Q75", "Q90", "Q95"] + +CHUNK_SIZE: int = 100_000 + +# ============================================================================= +# MIXTURE QUANTILE COMPUTATION +# ============================================================================= + + +def _interp_cdf_on_grid( + model_q: np.ndarray, + quantile_levels: np.ndarray, + grid: np.ndarray, +) -> np.ndarray: + """Interpolate CDF values on a discharge grid for one model (vectorized). + + Parameters + ---------- + model_q : shape (n_samples, n_quantiles) + quantile_levels : shape (n_quantiles,) + grid : shape (n_samples, n_grid) + + Returns + ------- + cdf_values : shape (n_samples, n_grid) + """ + n_q = model_q.shape[1] + + # Find bracketing quantile index for each grid value via broadcasting. + # idx = number of quantile breakpoints <= grid value (used as upper bracket). + # Then linear interpolation between bracketing quantile levels below. + idx = (grid[:, :, None] >= model_q[:, None, :]).sum(axis=2) # (n_samples, n_grid) + + idx_hi = np.clip(idx, 1, n_q - 1) + idx_lo = idx_hi - 1 + + rows = np.arange(model_q.shape[0])[:, None] + + x_lo = model_q[rows, idx_lo] + x_hi = model_q[rows, idx_hi] + y_lo = quantile_levels[idx_lo] + y_hi = quantile_levels[idx_hi] + + dx = x_hi - x_lo + t = np.where(dx > 0, (grid - x_lo) / dx, 0.5) + t = np.clip(t, 0.0, 1.0) + + cdf = y_lo + t * (y_hi - y_lo) + + # Clamp: below min quantile -> 0, above max quantile -> 1 + cdf = np.where(grid < model_q[:, 0:1], 0.0, cdf) + cdf = np.where(grid > model_q[:, -1:], 1.0, cdf) + + return cdf + + +def _invert_cdf( + avg_cdf: np.ndarray, + grid: np.ndarray, + target_levels: np.ndarray, +) -> np.ndarray: + """Invert average CDF to get mixture quantiles (vectorized). + + Parameters + ---------- + avg_cdf : shape (n_samples, n_grid) - monotonically non-decreasing + grid : shape (n_samples, n_grid) + target_levels : shape (n_targets,) + + Returns + ------- + quantiles : shape (n_samples, n_targets) + """ + n_samples, n_grid = avg_cdf.shape + n_targets = len(target_levels) + result = np.empty((n_samples, n_targets)) + + for j, level in enumerate(target_levels): + # For each sample, find index where CDF first exceeds level + # (n_samples, n_grid) < level -> bool, sum gives index of first exceedance + idx = (avg_cdf < level).sum(axis=1) # (n_samples,) + idx_hi = np.clip(idx, 1, n_grid - 1) + idx_lo = idx_hi - 1 + + rows = np.arange(n_samples) + cdf_lo = avg_cdf[rows, idx_lo] + cdf_hi = avg_cdf[rows, idx_hi] + grid_lo = grid[rows, idx_lo] + grid_hi = grid[rows, idx_hi] + + dcdf = cdf_hi - cdf_lo + t = np.where(dcdf > 0, (level - cdf_lo) / dcdf, 0.5) + t = np.clip(t, 0.0, 1.0) + + result[:, j] = grid_lo + t * (grid_hi - grid_lo) + + return result + + +def build_mixture_quantiles_vectorized( + model_quantiles: np.ndarray, + quantile_levels: np.ndarray, + n_grid: int = 200, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Build mixture quantiles from multiple models (fully vectorized). + + Parameters + ---------- + model_quantiles : shape (n_models, n_samples, n_quantiles) + quantile_levels : shape (n_quantiles,) + n_grid : grid resolution for CDF approximation + + Returns + ------- + mixture_quantiles : shape (n_samples, n_quantiles) + epistemic : shape (n_samples,) - mean std across all quantile levels + aleatoric : shape (n_samples,) - mean of per-model Q95-Q5 + """ + n_models, n_samples, _ = model_quantiles.shape + + # Enforce monotonicity per model + model_quantiles = np.maximum.accumulate(model_quantiles, axis=-1) + + # Epistemic: mean std across all quantile levels (captures tail disagreement) + # shape: std over models -> (n_samples, n_q), then mean over quantiles -> (n_samples,) + epistemic = np.mean(np.std(model_quantiles, axis=0), axis=-1) + + # Aleatoric: mean of per-model 90% interval width + aleatoric = np.mean(model_quantiles[:, :, -1] - model_quantiles[:, :, 0], axis=0) + + # Build per-sample grid + global_min = model_quantiles[:, :, 0].min(axis=0) - 1e-6 # (n_samples,) + global_max = model_quantiles[:, :, -1].max(axis=0) + 1e-6 # (n_samples,) + + # grid: (n_samples, n_grid) + t = np.linspace(0, 1, n_grid)[None, :] # (1, n_grid) + grid = global_min[:, None] + t * (global_max - global_min)[:, None] + + # Compute and average CDFs + avg_cdf = np.zeros((n_samples, n_grid)) + for m in range(n_models): + avg_cdf += _interp_cdf_on_grid(model_quantiles[m], quantile_levels, grid) + avg_cdf /= n_models + + # Invert average CDF at target quantile levels + mixture_quantiles = _invert_cdf(avg_cdf, grid, quantile_levels) + + # Final monotonicity enforcement on mixture output + mixture_quantiles = np.maximum.accumulate(mixture_quantiles, axis=-1) + + return mixture_quantiles, epistemic, aleatoric + + +# ============================================================================= +# DATA LOADING +# ============================================================================= + + +def load_and_align_predictions( + member_folders: list[Path], + split: str, + quantile_cols: list[str], +) -> tuple[pd.DataFrame, np.ndarray]: + """Load predictions from all members and align on common index. + + Returns + ------- + index_df : DataFrame with [date, code, forecast_step] for common rows + model_quantiles : shape (n_models, n_common_rows, n_quantiles) + """ + key_cols = ["date", "code", "forecast_step"] + + dfs = [] + for folder in member_folders: + path = folder / "predictions" / f"{split}_predictions.parquet" + if not path.exists(): + raise FileNotFoundError(f"Predictions not found: {path}") + df = pd.read_parquet(path) + df["date"] = pd.to_datetime(df["date"]) + dfs.append(df) + logger.info(f"Loaded {len(df)} rows from {folder.name}") + + # Find common (date, code, forecast_step) across all members + common_keys = reduce( + lambda a, b: pd.merge(a, b, on=key_cols, how="inner")[key_cols], + [df[key_cols] for df in dfs], + ) + common_keys = common_keys.sort_values(key_cols).reset_index(drop=True) + logger.info(f"Common index: {len(common_keys)} rows across {len(dfs)} members") + + # Align each member to common index and extract quantiles + arrays = [] + for df in dfs: + merged = pd.merge(common_keys, df, on=key_cols, how="inner") + merged = merged.sort_values(key_cols).reset_index(drop=True) + arrays.append(merged[quantile_cols].values) + + model_quantiles = np.stack(arrays, axis=0) + return common_keys, model_quantiles + + +def build_paths_dict(config: RunConfig) -> dict: + return { + "path_rivers": config.path_rivers, + "path_forcing": config.path_forcing, + "path_to_operational_forcing": config.path_to_operational_forcing, + "path_to_hindcast_forcing": config.path_to_hindcast_forcing, + "HRU_forcing": config.HRU_forcing, + "path_static": config.path_static, + "path_to_sla": config.path_to_sla, + "path_to_nir": config.path_to_nir, + "path_to_swe": config.path_to_swe, + "path_to_hs": config.path_to_hs, + "path_to_rof": config.path_to_rof, + } + + +def load_observations(member_folder: Path) -> pd.DataFrame: + """Load observations using config from an ensemble member.""" + run_config, _ = load_run_artifacts(member_folder) + + if getattr(run_config, "data_source", "legacy") == "caravan" and run_config.caravan: + from st_forecast.config.data_config import CaravanConfig + + caravan_config = CaravanConfig( + base_path=run_config.caravan["base_path"], + region=run_config.caravan["region"], + n_basins=run_config.caravan.get("n_basins"), + basin_ids=run_config.caravan.get("basin_ids"), + start_date=run_config.caravan.get("start_date"), + end_date=run_config.caravan.get("end_date"), + ) + collector = CaravanDataCollector(caravan_config) + else: + paths_dict = build_paths_dict(run_config) + collector = RawDataCollector(paths_dict, run_config.region) + + return collector.collect_discharge() + + +# ============================================================================= +# VISUALIZATION +# ============================================================================= + + +def plot_ensemble_uncertainty( + predictions_df: pd.DataFrame, + observations_df: pd.DataFrame, + output_folder: Path, + basin_codes: list[str | int], + forecast_horizon: int | None = None, +) -> None: + """Plot ensemble forecasts with epistemic uncertainty shown separately. + + For each basin, creates a rolling forecast plot with: + - Total uncertainty band (Q5-Q95, light blue) + - Inner uncertainty band (Q25-Q75, medium blue) + - Median forecast line (Q50, blue) + - Epistemic uncertainty band (Q50 ± epistemic, orange) + - Observed discharge (black) + """ + output_folder.mkdir(parents=True, exist_ok=True) + + if forecast_horizon is None: + forecast_horizon = int(predictions_df["forecast_step"].max()) + + for basin_code in basin_codes: + basin_pred = predictions_df[predictions_df["code"] == basin_code].copy() + basin_obs = observations_df[observations_df["code"] == basin_code].copy() + + if basin_pred.empty or basin_obs.empty: + logger.warning(f"No data for basin {basin_code}") + continue + + basin_pred["date"] = pd.to_datetime(basin_pred["date"]) + basin_obs["date"] = pd.to_datetime(basin_obs["date"]) + + # Build rolling forecast trace + forecast_starts = sorted( + basin_pred[basin_pred["forecast_step"] == 1]["date"].unique() + ) + + rolling_rows = [] + for start_date in forecast_starts: + for step in range(1, forecast_horizon + 1): + target_date = start_date + pd.Timedelta(days=step - 1) + row = basin_pred[ + (basin_pred["date"] == target_date) + & (basin_pred["forecast_step"] == step) + ] + if not row.empty: + rolling_rows.append(row.iloc[0]) + + if not rolling_rows: + continue + + rolling_df = ( + pd.DataFrame(rolling_rows) + .sort_values("date") + .drop_duplicates(subset=["date"], keep="first") + ) + + # Filter observations to prediction range + date_min, date_max = rolling_df["date"].min(), rolling_df["date"].max() + basin_obs = basin_obs[ + (basin_obs["date"] >= date_min) & (basin_obs["date"] <= date_max) + ].sort_values("date") + + fig, axes = plt.subplots(2, 1, figsize=(14, 10), sharex=True) + + dates = rolling_df["date"] + + # --- Top panel: total uncertainty --- + ax = axes[0] + ax.plot( + basin_obs["date"], + basin_obs["discharge"], + color="black", + linewidth=1.5, + label="Observed", + zorder=5, + ) + + ax.fill_between( + dates, + rolling_df["Q5"], + rolling_df["Q95"], + alpha=0.15, + color=TOL[0], + label="Q5-Q95 (total)", + ) + ax.fill_between( + dates, + rolling_df["Q25"], + rolling_df["Q75"], + alpha=0.3, + color=TOL[0], + label="Q25-Q75", + ) + ax.plot( + dates, + rolling_df["Q50"], + color=TOL[0], + linewidth=1.2, + label="Q50 (Median)", + ) + + ax.set_ylabel("Discharge [m³/s]") + ax.set_title(f"Ensemble Forecast — Basin {basin_code}") + ax.legend(loc="upper right") + ax.grid(True, alpha=0.3) + + # --- Bottom panel: epistemic uncertainty --- + ax = axes[1] + ax.plot( + basin_obs["date"], + basin_obs["discharge"], + color="black", + linewidth=1.5, + label="Observed", + zorder=5, + ) + ax.plot( + dates, + rolling_df["Q50"], + color=TOL[0], + linewidth=1.2, + label="Q50 (Median)", + ) + + if "epistemic" in rolling_df.columns: + epi_lo = rolling_df["Q50"] - rolling_df["epistemic"] + epi_hi = rolling_df["Q50"] + rolling_df["epistemic"] + ax.fill_between( + dates, + epi_lo, + epi_hi, + alpha=0.3, + color=TOL[1], + label="Epistemic (±1σ model disagreement)", + ) + + ax.set_xlabel("Date") + ax.set_ylabel("Discharge [m³/s]") + ax.set_title(f"Epistemic Uncertainty — Basin {basin_code}") + ax.legend(loc="upper right") + ax.grid(True, alpha=0.3) + + plt.tight_layout() + out_path = output_folder / f"ensemble_uncertainty_{basin_code}.png" + plt.savefig(out_path, bbox_inches="tight", dpi=300) + plt.close() + logger.info(f"Saved epistemic uncertainty plot for basin {basin_code}") + + +# ============================================================================= +# MAIN +# ============================================================================= + + +def main() -> None: + logger.info(f"Creating ensemble from {len(ENSEMBLE_MEMBERS)} members") + + # 1. Load and align predictions + index_df, model_quantiles = load_and_align_predictions( + ENSEMBLE_MEMBERS, SPLIT, QUANTILE_COLS + ) + + n_models, n_samples, n_q = model_quantiles.shape + logger.info( + f"Stacked quantiles: {n_models} models × {n_samples} samples × {n_q} quantiles" + ) + + # 2. Compute mixture quantiles in chunks for memory efficiency + quantile_levels = np.array(QUANTILE_LEVELS) + all_mixture = np.empty((n_samples, n_q)) + all_epistemic = np.empty(n_samples) + all_aleatoric = np.empty(n_samples) + + n_chunks = (n_samples + CHUNK_SIZE - 1) // CHUNK_SIZE + for i in range(n_chunks): + start = i * CHUNK_SIZE + end = min((i + 1) * CHUNK_SIZE, n_samples) + chunk = model_quantiles[:, start:end, :] + + mix_q, epi, ale = build_mixture_quantiles_vectorized(chunk, quantile_levels) + + all_mixture[start:end] = mix_q + all_epistemic[start:end] = epi + all_aleatoric[start:end] = ale + + if n_chunks > 1: + logger.info(f" Chunk {i + 1}/{n_chunks} done ({end - start} samples)") + + logger.info("Mixture quantile computation complete") + + # 3. Build ensemble DataFrame + ensemble_df = index_df.copy() + for j, col in enumerate(QUANTILE_COLS): + ensemble_df[col] = all_mixture[:, j] + ensemble_df["epistemic"] = all_epistemic + ensemble_df["aleatoric"] = all_aleatoric + + # 4. Save predictions + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + save_predictions(ensemble_df, OUTPUT_DIR, SPLIT) + + # 5. Load observations and evaluate + logger.info("Loading observations...") + observations_df = load_observations(ENSEMBLE_MEMBERS[0]) + logger.info(f"Loaded {len(observations_df)} observation rows") + + run_config, _ = load_run_artifacts(ENSEMBLE_MEMBERS[0]) + + eval_config = EvaluationConfig( + output_folder=OUTPUT_DIR / "metrics", + split=SPLIT, + quantile_levels=QUANTILE_LEVELS, + quantile_cols=QUANTILE_COLS, + ) + + # Drop extra columns before evaluation (evaluator doesn't expect them) + eval_df = ensemble_df.drop(columns=["epistemic", "aleatoric"]) + result = evaluate(eval_df, observations_df, eval_config) + + median_nse = result.performance_metrics["NSE"].median() + logger.info(f"Ensemble median NSE: {median_nse:.4f}") + + if result.probabilistic_global: + logger.info( + f"Ensemble global CRPS: {result.probabilistic_global.get('CRPS', 'N/A')}" + ) + + # 6. Standard visualizations (metrics, calibration, flood) + logger.info("Generating standard visualizations...") + viz_config = VisualizationConfig( + run_folder=OUTPUT_DIR, + split=SPLIT, + output_folder=OUTPUT_DIR / "figures", + quantile_levels=QUANTILE_LEVELS, + ) + plots = generate_visualizations( + viz_config, ["metrics", "probabilistic", "flood"], observations_df + ) + logger.info(f"Generated {len(plots)} standard plots") + + # 7. Ensemble-specific uncertainty plots + logger.info("Generating ensemble uncertainty plots...") + metrics_path = OUTPUT_DIR / "metrics" / f"{SPLIT}_metrics.csv" + if metrics_path.exists(): + metrics_df = pd.read_csv(metrics_path) + basin_codes = select_representative_basins(metrics_df) + else: + basin_codes = ensemble_df["code"].unique()[:4].tolist() + + figures_dir = OUTPUT_DIR / "figures" + + # Standard forecast plots + from st_forecast.visualization import plot_example_forecasts + + plot_example_forecasts( + ensemble_df, + observations_df, + figures_dir, + basin_codes, + QUANTILE_COLS, + ) + + # Epistemic uncertainty plots + plot_ensemble_uncertainty( + ensemble_df, + observations_df, + figures_dir, + basin_codes, + ) + + logger.info(f"Ensemble creation complete. Output: {OUTPUT_DIR}") + + +if __name__ == "__main__": + main() diff --git a/scripts/explain_tft.py b/scripts/explain_tft.py index 4170f18..5e539ba 100644 --- a/scripts/explain_tft.py +++ b/scripts/explain_tft.py @@ -14,6 +14,7 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +import seaborn as sns from st_forecast.config.data_config import CaravanConfig from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector @@ -194,24 +195,44 @@ def _plot_importance_boxplot( data = imp_df[feature_cols] # Sort by median descending medians = data.median().sort_values(ascending=False) - data = data[medians.index] + ordered_cols = list(medians.index) + + # Melt to long form for seaborn + long_df = data[ordered_cols].melt(var_name="Feature", value_name="Importance") fig, ax = plt.subplots(figsize=figsize(1.0, 0.5)) - bp = ax.boxplot( - data.values, - labels=list(data.columns), - patch_artist=True, - medianprops={"color": "black", "linewidth": 1.2}, + + sns.boxplot( + data=long_df, + x="Feature", + y="Importance", + color=TOL[0], + ax=ax, + showfliers=False, + order=ordered_cols, + boxprops={"alpha": 0.4}, + whiskerprops={"alpha": 0.4}, + capprops={"alpha": 0.4}, + medianprops={"alpha": 0.8}, + ) + + sns.stripplot( + data=long_df, + x="Feature", + y="Importance", + color=TOL[0], + ax=ax, + order=ordered_cols, + alpha=0.4, + size=3, ) - for patch in bp["boxes"]: - patch.set_facecolor(TOL[0]) - patch.set_alpha(0.7) - ax.set_ylabel("Importance (%)") - ax.set_title(title) + ax.set_ylabel("Importance [%]") + ax.set_xlabel("") + ax.set_title("") plt.xticks(rotation=45, ha="right") plt.tight_layout() - fig.savefig(output_path) + fig.savefig(output_path, dpi=300, bbox_inches="tight") plt.close(fig) logger.info(f"Saved {output_path}") @@ -245,14 +266,15 @@ def _plot_attention( p10 = np.percentile(all_profiles, 10, axis=0) p90 = np.percentile(all_profiles, 90, axis=0) - ax.plot(x, mean_line, color=TOL[0], linewidth=1.5, label="Mean") - ax.fill_between(x, p10, p90, color=TOL[0], alpha=0.2, label="10th-90th pctl") - ax.set_xlabel("Relative time step") - ax.set_ylabel("Attention weight") - ax.set_title("Encoder Attention Weights") - ax.legend() + ax.plot(x, mean_line, color=TOL[0], linewidth=2, label="Mean") + ax.fill_between(x, p10, p90, color=TOL[0], alpha=0.2, label="10th–90th pctl") + ax.set_xlabel("Relative Time Step [days]") + ax.set_ylabel("Attention Weight [-]") + ax.set_title("") + ax.legend(bbox_to_anchor=(1.02, 1), loc="upper left") + ax.grid(True, alpha=0.3) plt.tight_layout() - fig.savefig(output_path) + fig.savefig(output_path, dpi=300, bbox_inches="tight") plt.close(fig) logger.info(f"Saved {output_path}") @@ -262,7 +284,7 @@ def main() -> None: parser.add_argument("model_folder", type=Path, help="Path to experiment folder") args = parser.parse_args() - use_style("paper") + use_style("conference") model_folder = args.model_folder logger.info(f"Loading artifacts from {model_folder}") diff --git a/st_forecast/custom_models/ExoLSTM.py b/st_forecast/custom_models/ExoLSTM.py index cb53e1b..ca95387 100644 --- a/st_forecast/custom_models/ExoLSTM.py +++ b/st_forecast/custom_models/ExoLSTM.py @@ -15,7 +15,7 @@ from darts.logging import get_logger, raise_log from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel @@ -28,7 +28,7 @@ logger = get_logger(__name__) -class _ExoLSTMModule(PLMixedCovariatesModule): +class _ExoLSTMModule(PLForecastingModule): def __init__( self, input_dim: int, diff --git a/st_forecast/custom_models/ExoMamba.py b/st_forecast/custom_models/ExoMamba.py index 397e2bc..d84fda9 100644 --- a/st_forecast/custom_models/ExoMamba.py +++ b/st_forecast/custom_models/ExoMamba.py @@ -15,7 +15,7 @@ from darts.logging import get_logger, raise_log from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel @@ -35,7 +35,7 @@ logger = get_logger(__name__) -class _ExoMambaModule(PLMixedCovariatesModule): +class _ExoMambaModule(PLForecastingModule): def __init__( self, input_dim: int, diff --git a/st_forecast/custom_models/FANMixer.py b/st_forecast/custom_models/FANMixer.py index 7c1fce2..09090f0 100644 --- a/st_forecast/custom_models/FANMixer.py +++ b/st_forecast/custom_models/FANMixer.py @@ -13,7 +13,7 @@ from darts.logging import get_logger, raise_log from darts.models.components import layer_norm_variants from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel @@ -327,7 +327,7 @@ def forward( return x -class _FANMixerModule(PLMixedCovariatesModule): +class _FANMixerModule(PLForecastingModule): """ FANMixer module for use within a Darts forecasting model. This extends the TSMixerModule by replacing the time mixing with FAN-based mixing. diff --git a/st_forecast/custom_models/lstm/module.py b/st_forecast/custom_models/lstm/module.py index 57f811a..49b649d 100644 --- a/st_forecast/custom_models/lstm/module.py +++ b/st_forecast/custom_models/lstm/module.py @@ -3,7 +3,7 @@ from darts.logging import get_logger from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.utils.torch import MonteCarloDropout @@ -11,7 +11,7 @@ logger = get_logger(__name__) -class _LSTMModule(PLMixedCovariatesModule): +class _LSTMModule(PLForecastingModule): def __init__( self, output_dim: int, diff --git a/st_forecast/custom_models/lstm_enc_dec/module.py b/st_forecast/custom_models/lstm_enc_dec/module.py index ff7e0ef..ec1a7b2 100644 --- a/st_forecast/custom_models/lstm_enc_dec/module.py +++ b/st_forecast/custom_models/lstm_enc_dec/module.py @@ -3,7 +3,7 @@ from darts.logging import get_logger, raise_log from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.utils.torch import MonteCarloDropout @@ -11,7 +11,7 @@ logger = get_logger(__name__) -class _LSTMEncoderDecoderModule(PLMixedCovariatesModule): +class _LSTMEncoderDecoderModule(PLForecastingModule): def __init__( self, output_dim: int, diff --git a/st_forecast/custom_models/lstm_enc_mlp/module.py b/st_forecast/custom_models/lstm_enc_mlp/module.py index 6df1306..3e09d67 100644 --- a/st_forecast/custom_models/lstm_enc_mlp/module.py +++ b/st_forecast/custom_models/lstm_enc_mlp/module.py @@ -3,7 +3,7 @@ from darts.logging import get_logger, raise_log from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.utils.torch import MonteCarloDropout @@ -47,7 +47,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return out -class _LSTMEncMLPModule(PLMixedCovariatesModule): +class _LSTMEncMLPModule(PLForecastingModule): def __init__( self, output_dim: int, diff --git a/st_forecast/custom_models/mamba_enc_dec/module.py b/st_forecast/custom_models/mamba_enc_dec/module.py index 36d5a22..8e02c8b 100644 --- a/st_forecast/custom_models/mamba_enc_dec/module.py +++ b/st_forecast/custom_models/mamba_enc_dec/module.py @@ -3,7 +3,7 @@ from darts.logging import get_logger, raise_log from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.utils.torch import MonteCarloDropout @@ -18,7 +18,7 @@ logger = get_logger(__name__) -class _MambaEncoderDecoderModule(PLMixedCovariatesModule): +class _MambaEncoderDecoderModule(PLForecastingModule): def __init__( self, output_dim: int, diff --git a/st_forecast/custom_models/unified_temporal/module.py b/st_forecast/custom_models/unified_temporal/module.py index a0b4a26..5c4c727 100644 --- a/st_forecast/custom_models/unified_temporal/module.py +++ b/st_forecast/custom_models/unified_temporal/module.py @@ -1,4 +1,4 @@ -"""UnifiedTemporalModule - PLMixedCovariatesModule for UnifiedTemporalModel.""" +"""UnifiedTemporalModule - PLForecastingModule for UnifiedTemporalModel.""" from typing import Optional @@ -7,7 +7,7 @@ from darts.logging import get_logger, raise_log from darts.models.forecasting.pl_forecasting_module import ( - PLMixedCovariatesModule, + PLForecastingModule, io_processor, ) from darts.utils.torch import MonteCarloDropout @@ -20,7 +20,7 @@ logger = get_logger(__name__) -class _UnifiedTemporalModule(PLMixedCovariatesModule): +class _UnifiedTemporalModule(PLForecastingModule): def __init__( self, input_dim: int, diff --git a/st_forecast/evaluate_model.py b/st_forecast/evaluate_model.py index caeb2dc..15c601d 100644 --- a/st_forecast/evaluate_model.py +++ b/st_forecast/evaluate_model.py @@ -117,7 +117,7 @@ def create_prediction_df( data = { "date": predictions.time_index, **{ - f"Q{int(q * 100)}": predictions.quantile_timeseries(q) + f"Q{int(q * 100)}": predictions.quantile(q) .values() .flatten() for q in quantiles @@ -258,13 +258,13 @@ def eval_model( # Store predictions if probabilistic: if code not in predictions: - predictions[code] = [backtest.quantile_timeseries(0.5)] - q_low[code] = [backtest.quantile_timeseries(0.1)] - q_high[code] = [backtest.quantile_timeseries(0.9)] + predictions[code] = [backtest.quantile(0.5)] + q_low[code] = [backtest.quantile(0.1)] + q_high[code] = [backtest.quantile(0.9)] else: - predictions[code].append(backtest.quantile_timeseries(0.5)) - q_low[code].append(backtest.quantile_timeseries(0.1)) - q_high[code].append(backtest.quantile_timeseries(0.9)) + predictions[code].append(backtest.quantile(0.5)) + q_low[code].append(backtest.quantile(0.1)) + q_high[code].append(backtest.quantile(0.9)) else: if code not in predictions: predictions[code] = [backtest] diff --git a/st_forecast/model_helper.py b/st_forecast/model_helper.py index 10e3cc7..0308d4f 100644 --- a/st_forecast/model_helper.py +++ b/st_forecast/model_helper.py @@ -119,6 +119,10 @@ def initialize_model(config: RunConfig) -> tuple[object, TrainingLossLogger]: "force_reset": True, } + # RNNModel doesn't accept output_chunk_length (forced to 1 internally) + if config.model_type == "RNN": + common_kwargs.pop("output_chunk_length") + # Model-specific kwargs model_specific = _get_model_specific_kwargs(config) model_kwargs = {**common_kwargs, **model_specific} @@ -162,6 +166,10 @@ def _get_model_class(model_type: str): from st_forecast.custom_models.lstm import LSTMModel return LSTMModel + case "RNN": + from darts.models import RNNModel + + return RNNModel case _: raise ValueError( f"Unknown model_type: {model_type}. " @@ -288,6 +296,17 @@ def _get_model_specific_kwargs(config: RunConfig) -> dict: "use_reversible_instance_norm": use_rin and use_past_target, "obs_mask_prob": extra.get("obs_mask_prob", 0.0), } + case "RNN": + return { + "model": extra.get("rnn_model", "LSTM"), + "hidden_dim": config.hidden_size, + "n_rnn_layers": extra.get("n_rnn_layers", 2), + "dropout": config.dropout, + "training_length": extra.get( + "training_length", + config.input_chunk_length + config.forecast_horizon, + ), + } case "LSTMEncMLP": use_past_target = extra.get("use_past_target", True) return { diff --git a/st_forecast/pipeline/artifacts.py b/st_forecast/pipeline/artifacts.py index 9243c33..3ae6095 100644 --- a/st_forecast/pipeline/artifacts.py +++ b/st_forecast/pipeline/artifacts.py @@ -19,6 +19,7 @@ "mambaencdec": "MambaEncDec", "lstmencmlp": "LSTMEncMLP", "lstm": "LSTM", + "rnn": "RNN", "persistence": "Persistence", } diff --git a/st_forecast/pipeline/inference.py b/st_forecast/pipeline/inference.py index 28a9ac6..179c07c 100644 --- a/st_forecast/pipeline/inference.py +++ b/st_forecast/pipeline/inference.py @@ -83,6 +83,10 @@ def load_model(config: RunConfig, run_folder: Path) -> object: from st_forecast.custom_models.lstm import LSTMModel return LSTMModel.load(str(model_path)) + elif model_type == "RNN": + from darts.models import RNNModel + + return RNNModel.load(str(model_path)) else: raise ValueError(f"Unknown model type: {model_type}") @@ -112,7 +116,7 @@ def create_prediction_df( data = { date_col: predictions.time_index, **{ - f"Q{int(q * 100)}": predictions.quantile_timeseries(q) + f"Q{int(q * 100)}": predictions.quantile(q) .values() .flatten() for q in quantiles @@ -216,7 +220,7 @@ def _process_basin( steps.append(np.arange(1, n + 1)) for q in quantiles: col = f"Q{int(q * 100)}" - quantile_data[col].append(ts.quantile_timeseries(q).values().flatten()) + quantile_data[col].append(ts.quantile(q).values().flatten()) return { "dates": np.concatenate(dates), diff --git a/st_forecast/pipeline/training.py b/st_forecast/pipeline/training.py index 6bcb19b..1ddd011 100644 --- a/st_forecast/pipeline/training.py +++ b/st_forecast/pipeline/training.py @@ -63,6 +63,10 @@ def _get_model_class(model_type: str): return MambaEncoderDecoderModel case "LSTM": return LSTMModel + case "RNN": + from darts.models import RNNModel + + return RNNModel case _: raise ValueError( f"Unknown model_type: {model_type}. " diff --git a/st_forecast/visualization/__init__.py b/st_forecast/visualization/__init__.py index ee21bbf..422192e 100644 --- a/st_forecast/visualization/__init__.py +++ b/st_forecast/visualization/__init__.py @@ -15,7 +15,9 @@ TOL, PAPER, PRESENTATION, + CONFERENCE, DEFAULT, + PERSISTENCE_COLOR, LEAD_TIME_COLORS, RETURN_PERIOD_COLORS, RETURN_PERIOD_LINESTYLES, @@ -70,7 +72,9 @@ # Multi-run comparison from .compare_runs import ( plot_nse_comparison, + plot_nrmse_comparison, plot_calibration_comparison, + plot_crps_comparison, plot_flood_f1_comparison, generate_comparison_plots, ) @@ -233,7 +237,9 @@ def generate_visualizations( "TOL", "PAPER", "PRESENTATION", + "CONFERENCE", "DEFAULT", + "PERSISTENCE_COLOR", "LEAD_TIME_COLORS", "RETURN_PERIOD_COLORS", "RETURN_PERIOD_LINESTYLES", @@ -271,7 +277,9 @@ def generate_visualizations( "plot_time_series", # Compare runs "plot_nse_comparison", + "plot_nrmse_comparison", "plot_calibration_comparison", + "plot_crps_comparison", "plot_flood_f1_comparison", "generate_comparison_plots", # Orchestrator diff --git a/st_forecast/visualization/_helpers.py b/st_forecast/visualization/_helpers.py index 164f107..08e0229 100644 --- a/st_forecast/visualization/_helpers.py +++ b/st_forecast/visualization/_helpers.py @@ -121,13 +121,14 @@ def load_multiple_run_metrics( for key, df in run_data.items(): if not df.empty: df = df.copy() - df["run"] = label + df["run"] = label all_metrics[key].append(df) result = {} for key, dfs in all_metrics.items(): if dfs: - result[key] = pd.concat(dfs, ignore_index=True) + combined = pd.concat(dfs, ignore_index=True) + result[key] = combined else: result[key] = pd.DataFrame() diff --git a/st_forecast/visualization/_styles.py b/st_forecast/visualization/_styles.py index 76022d4..f34db3e 100644 --- a/st_forecast/visualization/_styles.py +++ b/st_forecast/visualization/_styles.py @@ -80,6 +80,30 @@ "lines.markersize": 8, } +CONFERENCE: dict[str, object] = { + "font.family": "sans-serif", + "font.size": 16, + "axes.labelsize": 18, + "axes.titlesize": 18, + "xtick.labelsize": 14, + "ytick.labelsize": 14, + "legend.fontsize": 14, + "figure.dpi": 100, + "savefig.dpi": 300, + "savefig.bbox": "tight", + "axes.linewidth": 1.2, + "lines.linewidth": 2.0, + "lines.markersize": 8, +} + +PERSISTENCE_COLOR = "#444444" + +# Explicit model color overrides for visual distinction +MODEL_COLORS: dict[str, str] = { + "lstm": "#4477AA", # blue + "ensemble": "#AA3377", # purple +} + # Default style (current behavior) DEFAULT: dict[str, object] = { "font.size": 14, @@ -96,6 +120,7 @@ _STYLES = { "paper": PAPER, "presentation": PRESENTATION, + "conference": CONFERENCE, "default": DEFAULT, } diff --git a/st_forecast/visualization/compare_runs.py b/st_forecast/visualization/compare_runs.py index c3bf519..45f0a68 100644 --- a/st_forecast/visualization/compare_runs.py +++ b/st_forecast/visualization/compare_runs.py @@ -1,4 +1,4 @@ -"""Multi-run comparison visualizations.""" +"""Multi-run comparison visualizations (conference-ready).""" from __future__ import annotations @@ -11,24 +11,40 @@ from ._config import RunComparisonConfig from ._helpers import load_multiple_run_metrics -from ._styles import set_palette +from ._styles import MODEL_COLORS, PERSISTENCE_COLOR, TOL, use_style logger = logging.getLogger(__name__) +def _build_palette( + run_labels: list[str], + persistence_label: str = "Persistence", +) -> dict[str, str]: + reserved = set(MODEL_COLORS.values()) + available_tol = [c for c in TOL if c not in reserved] + tol_idx = 0 + palette: dict[str, str] = {} + for label in run_labels: + if label.lower() == persistence_label.lower(): + palette[label] = PERSISTENCE_COLOR + elif label.lower() in MODEL_COLORS: + palette[label] = MODEL_COLORS[label.lower()] + else: + palette[label] = available_tol[tol_idx % len(available_tol)] + tol_idx += 1 + return palette + + +def _apply_conference_style() -> None: + use_style("conference") + + def plot_nse_comparison( config: RunComparisonConfig, output_path: Path | None = None, ) -> Path | None: - """Plot NSE distribution comparison across runs as boxplots with jittered points. - - Args: - config: Run comparison configuration. - output_path: Optional path to save the figure. + _apply_conference_style() - Returns: - Output path if saved, None otherwise. - """ data = load_multiple_run_metrics( config.run_folders, config.run_labels, config.split ) @@ -42,8 +58,7 @@ def plot_nse_comparison( logger.warning("No forecast_step column in metrics") return None - run_colors = set_palette(len(config.run_labels)) - palette = dict(zip(config.run_labels, run_colors)) + palette = _build_palette(config.run_labels) fig, ax = plt.subplots(figsize=(12, 6)) @@ -74,11 +89,41 @@ def plot_nse_comparison( legend=False, ) - ax.set_xlabel("Forecast Step (days)") - ax.set_ylabel("NSE") - ax.set_title("NSE Distribution by Forecast Step") - ax.legend(title="Run", bbox_to_anchor=(1.02, 1), loc="upper left") - ax.axhline(y=0, color="gray", linestyle="--", linewidth=0.8, alpha=0.7) + ax.set_xlabel("Forecast Step [days]") + ax.set_ylabel("NSE [-]") + ax.set_title("") + ax.set_ylim(0.4, 1.0) + ax.legend(title=None, bbox_to_anchor=(1.02, 1), loc="upper left") + + # Count basins below 0.4 per model and forecast step + below_counts = ( + metrics_df[metrics_df["NSE"] < 0.4] + .groupby(["forecast_step", "run"]) + .size() + .reset_index(name="n_below") + ) + + if not below_counts.empty: + steps = sorted(metrics_df["forecast_step"].unique()) + runs = [lbl for lbl in config.run_labels if lbl in metrics_df["run"].unique()] + n_runs = len(runs) + + for _, row in below_counts.iterrows(): + step_idx = steps.index(row["forecast_step"]) + run_idx = runs.index(row["run"]) + # Offset within the group (centered dodge like seaborn) + width = 0.8 / n_runs + offset = (run_idx - (n_runs - 1) / 2) * width + ax.text( + step_idx + offset, + 0.405, + str(row["n_below"]), + ha="center", + va="bottom", + fontsize=7, + color=palette.get(row["run"], "black"), + fontweight="bold", + ) plt.tight_layout() @@ -96,15 +141,8 @@ def plot_nrmse_comparison( config: RunComparisonConfig, output_path: Path | None = None, ) -> Path | None: - """Plot nRMSE distribution comparison across runs as boxplots with jittered points. - - Args: - config: Run comparison configuration. - output_path: Optional path to save the figure. + _apply_conference_style() - Returns: - Output path if saved, None otherwise. - """ data = load_multiple_run_metrics( config.run_folders, config.run_labels, config.split ) @@ -118,8 +156,7 @@ def plot_nrmse_comparison( logger.warning("No forecast_step column in metrics") return None - run_colors = set_palette(len(config.run_labels)) - palette = dict(zip(config.run_labels, run_colors)) + palette = _build_palette(config.run_labels) fig, ax = plt.subplots(figsize=(12, 6)) @@ -150,10 +187,10 @@ def plot_nrmse_comparison( legend=False, ) - ax.set_xlabel("Forecast Step (days)") - ax.set_ylabel("nRMSE") - ax.set_title("nRMSE Distribution by Forecast Step") - ax.legend(title="Run", bbox_to_anchor=(1.02, 1), loc="upper left") + ax.set_xlabel("Forecast Step [days]") + ax.set_ylabel("nRMSE [-]") + ax.set_title("") + ax.legend(title=None, bbox_to_anchor=(1.02, 1), loc="upper left") plt.tight_layout() @@ -170,16 +207,10 @@ def plot_nrmse_comparison( def plot_calibration_comparison( config: RunComparisonConfig, output_path: Path | None = None, + persistence_label: str = "Persistence", ) -> Path | None: - """Plot global exceedance calibration lines for each run. - - Args: - config: Run comparison configuration. - output_path: Optional path to save the figure. + _apply_conference_style() - Returns: - Output path if saved, None otherwise. - """ data = load_multiple_run_metrics( config.run_folders, config.run_labels, config.split ) @@ -189,7 +220,11 @@ def plot_calibration_comparison( logger.warning("No probabilistic metrics available for calibration comparison") return None - run_colors = set_palette(len(config.run_labels)) + palette = _build_palette(config.run_labels) + # Filter out persistence + labels_no_persistence = [ + lbl for lbl in config.run_labels if lbl.lower() != persistence_label.lower() + ] fig, ax = plt.subplots(figsize=(8, 8)) @@ -197,7 +232,7 @@ def plot_calibration_comparison( [0, 1], [0, 1], "k--", linewidth=1.5, label="Perfect Calibration", alpha=0.7 ) - for run_label, color in zip(config.run_labels, run_colors): + for run_label in labels_no_persistence: run_data = prob_df[prob_df["run"] == run_label] if run_data.empty: continue @@ -217,15 +252,16 @@ def plot_calibration_comparison( theoretical_rates, empirical_rates, marker="o", - color=color, - linewidth=2, + color=palette[run_label], + linewidth=0.8, + linestyle="--", markersize=8, label=run_label, ) ax.set_xlabel("Theoretical Exceedance Rate") ax.set_ylabel("Empirical Exceedance Rate") - ax.set_title("Probabilistic Calibration Comparison") + ax.set_title("") ax.set_xlim(0, 1) ax.set_ylim(0, 1) ax.set_aspect("equal") @@ -244,21 +280,92 @@ def plot_calibration_comparison( return None -def plot_flood_f1_comparison( +def plot_crps_comparison( config: RunComparisonConfig, output_path: Path | None = None, + persistence_label: str = "Persistence", ) -> Path | None: - """Plot F1 score vs forecast step for multiple return periods. + _apply_conference_style() + + data = load_multiple_run_metrics( + config.run_folders, config.run_labels, config.split + ) + prob_df = data.get("probabilistic", pd.DataFrame()) + + if prob_df.empty or "CRPS" not in prob_df.columns: + logger.warning("No CRPS metrics available for comparison") + return None + + if "forecast_step" not in prob_df.columns: + logger.warning("No forecast_step column in probabilistic metrics") + return None + + prob_df = prob_df.dropna(subset=["forecast_step"]).copy() + prob_df["forecast_step"] = prob_df["forecast_step"].astype(int).astype(str) + + # Filter out persistence + labels_no_persistence = [ + lbl for lbl in config.run_labels if lbl.lower() != persistence_label.lower() + ] + palette = _build_palette(config.run_labels) + filtered_df = prob_df[prob_df["run"].isin(labels_no_persistence)] - Creates a 2x2 subplot grid for return periods 2, 5, 10, 20 years. + if filtered_df.empty: + logger.warning("No non-persistence CRPS data available") + return None - Args: - config: Run comparison configuration. - output_path: Optional path to save the figure. + fig, ax = plt.subplots(figsize=(12, 6)) + + sns.boxplot( + data=filtered_df, + x="forecast_step", + y="CRPS", + hue="run", + palette={lbl: palette[lbl] for lbl in labels_no_persistence}, + ax=ax, + showfliers=False, + boxprops={"alpha": 0.4}, + whiskerprops={"alpha": 0.4}, + capprops={"alpha": 0.4}, + medianprops={"alpha": 0.8}, + ) + + sns.stripplot( + data=filtered_df, + x="forecast_step", + y="CRPS", + hue="run", + palette={lbl: palette[lbl] for lbl in labels_no_persistence}, + ax=ax, + dodge=True, + alpha=0.4, + size=3, + legend=False, + ) + + ax.set_xlabel("Forecast Step [days]") + ax.set_ylabel("CRPS [-]") + ax.set_title("") + ax.legend(title=None, bbox_to_anchor=(1.02, 1), loc="upper left") + + plt.tight_layout() + + if output_path is not None: + output_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_path, dpi=300, bbox_inches="tight") + plt.close(fig) + return output_path + + plt.close(fig) + return None + + +def plot_flood_f1_comparison( + config: RunComparisonConfig, + output_path: Path | None = None, +) -> Path | None: + _apply_conference_style() - Returns: - Output path if saved, None otherwise. - """ data = load_multiple_run_metrics( config.run_folders, config.run_labels, config.split ) @@ -275,13 +382,13 @@ def plot_flood_f1_comparison( return None return_periods = [2.0, 5.0, 10.0, 20.0] - run_colors = set_palette(len(config.run_labels)) + palette = _build_palette(config.run_labels) - fig, axes = plt.subplots(2, 2, figsize=(12, 10), sharex=True, sharey=True) - axes = axes.flatten() + fig, axes = plt.subplots(2, 2, figsize=(14, 10), sharex=True, sharey=True) + axes_flat = axes.flatten() for idx, rp in enumerate(return_periods): - ax = axes[idx] + ax = axes_flat[idx] rp_data = flood_df[flood_df["return_period"] == rp] if rp_data.empty: @@ -293,10 +400,9 @@ def plot_flood_f1_comparison( va="center", transform=ax.transAxes, ) - ax.set_title(f"Return Period: {rp:.0f} years") continue - for run_label, color in zip(config.run_labels, run_colors): + for run_label in config.run_labels: run_data = rp_data[rp_data["run"] == run_label] if run_data.empty: continue @@ -306,22 +412,27 @@ def plot_flood_f1_comparison( grouped["forecast_step"], grouped["F1"], marker="o", - color=color, + color=palette[run_label], linewidth=2, markersize=6, label=run_label, ) - ax.set_title(f"Return Period: {rp:.0f} years") ax.set_ylim(0, 1) ax.grid(True, alpha=0.3) + ax.annotate( + f"RP = {rp:.0f} yr", + xy=(0.03, 0.95), + xycoords="axes fraction", + va="top", + fontweight="bold", + ) if idx == 0: ax.legend(loc="upper right") - fig.supxlabel("Forecast Step (days)") - fig.supylabel("F1 Score") - fig.suptitle("Flood Detection F1 Score by Return Period", fontsize=14, y=1.02) + fig.supxlabel("Forecast Step [days]") + fig.supylabel("F1 Score [-]") plt.tight_layout() @@ -336,14 +447,6 @@ def plot_flood_f1_comparison( def generate_comparison_plots(config: RunComparisonConfig) -> dict[str, Path]: - """Generate all comparison plots for multiple runs. - - Args: - config: Run comparison configuration. - - Returns: - Dictionary mapping plot names to saved file paths. - """ config.output_folder.mkdir(parents=True, exist_ok=True) plots: dict[str, Path] = {} @@ -352,6 +455,7 @@ def generate_comparison_plots(config: RunComparisonConfig) -> dict[str, Path]: ("nse_comparison", plot_nse_comparison), ("nrmse_comparison", plot_nrmse_comparison), ("calibration_comparison", plot_calibration_comparison), + ("crps_comparison", plot_crps_comparison), ("flood_f1_comparison", plot_flood_f1_comparison), ] diff --git a/tests/test_inference.py b/tests/test_inference.py index 8cbd873..e0d21e0 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -29,7 +29,7 @@ def mock_timeseries_with_quantiles(self): dates = pd.date_range("2020-01-01", periods=5, freq="D") ts.time_index = dates - # Mock quantile_timeseries to return different values for each quantile + # Mock quantile to return different values for each quantile def mock_quantile(q): quantile_ts = Mock() # Different values for each quantile @@ -38,7 +38,7 @@ def mock_quantile(q): quantile_ts.flatten.return_value = values.flatten() return quantile_ts - ts.quantile_timeseries = mock_quantile + ts.quantile = mock_quantile return ts @pytest.fixture @@ -146,7 +146,7 @@ def mock_historical_forecasts(**kwargs): # Use side_effect to return a constant value for len() ts.__len__ = Mock(return_value=5) - # Mock quantile_timeseries - need to capture values in closure + # Mock quantile - need to capture values in closure def make_quantile_fn(basin_id, timestep): def quantile_fn(q): q_ts = Mock() @@ -159,7 +159,7 @@ def quantile_fn(q): return quantile_fn - ts.quantile_timeseries = make_quantile_fn(basin_idx, t) + ts.quantile = make_quantile_fn(basin_idx, t) timestep_results.append(ts) basin_results.append(timestep_results) diff --git a/uv.lock b/uv.lock index 40af75f..cb2756d 100644 --- a/uv.lock +++ b/uv.lock @@ -6,18 +6,6 @@ resolution-markers = [ "python_full_version < '3.12'", ] -[[package]] -name = "adagio" -version = "0.2.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "triad" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f8/d7/c02a080407e133cf404a2b63bb3de1495c65d7af0501c313731a545d39ca/adagio-0.2.6.tar.gz", hash = "sha256:0c32768f3aba0e05273b36f9420a482034f2510f059171040d7e98ba34128d7a", size = 23653, upload-time = "2024-08-14T07:34:14.851Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/40/3592ba5232475778ab690cdbfbc38e73886c26c361a82484b49fab427e60/adagio-0.2.6-py3-none-any.whl", hash = "sha256:1bb8317d41bfff8b11373bc03c9859ff166c498214bb2b7ce1e21638c0babb2c", size = 19073, upload-time = "2024-08-14T07:34:13.506Z" }, -] - [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -144,15 +132,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] -[[package]] -name = "appdirs" -version = "1.4.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/d8/05696357e0311f5b5c316d7b95f46c669dd9c15aaeecbb48c7d0aeb88c40/appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", size = 13470, upload-time = "2020-05-11T07:59:51.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566, upload-time = "2020-05-11T07:59:49.499Z" }, -] - [[package]] name = "attrs" version = "25.3.0" @@ -377,32 +356,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/58/bd257695f39d05594ca4ad60df5bcb7e32247f9951fd09a9b8edb82d1daa/contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77", size = 225315, upload-time = "2025-07-26T12:02:58.801Z" }, ] -[[package]] -name = "coreforecast" -version = "0.0.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/4c/d9cd9d490f19447a74fd3e18940305252afab5bba8b518971b448c22ad39/coreforecast-0.0.16.tar.gz", hash = "sha256:47d7efc4a03e736dc29a44184934cf7535371fcd8434c3f2a31b0d663b6d88ea", size = 2759924, upload-time = "2025-04-03T19:34:40.577Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/97/62a45e2134befca6348e1e6fcfd084fe0111cc6cc2e29828d93a309a9615/coreforecast-0.0.16-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:deaffe3990c7712f5372ffb53310a47c966f36b6431a62ffd8b65082e72d8c33", size = 262992, upload-time = "2025-04-03T19:34:08.428Z" }, - { url = "https://files.pythonhosted.org/packages/de/4d/e5b1412ecd0433e2d54773e0886e9e75d4eaa5102bd6091e5e6aa4d51a74/coreforecast-0.0.16-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3dbc0ad3fa4a94de43be8dee3ed258569a4b61f91ba7bbd7242d00f4a7e49009", size = 225854, upload-time = "2025-04-03T19:34:10.113Z" }, - { url = "https://files.pythonhosted.org/packages/0f/8e/6770f6fc2f167d83ef2add958779cfc3ef437e363603e4a481f382c19d5b/coreforecast-0.0.16-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b307f3a1408e8b6d696e79b45d2b61a9842b0fd626ee5cecb8b10025a9084e61", size = 262009, upload-time = "2025-04-03T19:34:11.847Z" }, - { url = "https://files.pythonhosted.org/packages/b8/bf/19c7375e840cd50365f976ac24e2746ad3b3c71ceb69c6ab81e6bc7acec7/coreforecast-0.0.16-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8cfd447f9fc2dbf7f13fca1b1fa2af2bd18643d8423042f63ee064dbb348b23", size = 285816, upload-time = "2025-04-03T19:34:13.518Z" }, - { url = "https://files.pythonhosted.org/packages/e5/20/45af22aaf7b4f1b66ec6f03b370a704579f7609a69475ae25c3608cbdfb0/coreforecast-0.0.16-cp311-cp311-win_amd64.whl", hash = "sha256:96c2ea3ee226f010bf5e0f93e7a57465d235e39bc9c12ee13fcb70b25ffaba00", size = 207684, upload-time = "2025-04-03T19:34:15.237Z" }, - { url = "https://files.pythonhosted.org/packages/0a/9f/4262ae919e83f45be32676ff82b96dbdf2e208fce9fe1b8eab09b10bd01a/coreforecast-0.0.16-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:58d8ce565b87c54359c10b427ec196d246bc5c729e107be9a30158678063c288", size = 259144, upload-time = "2025-04-03T19:34:16.993Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f1/0aaeea4ef13f1110029868667b2dcda0471e003e70333e4d2762feacecba/coreforecast-0.0.16-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:00c7f8061acf10c1e0aca1cca1d958991a030925600386491e1c89658305b9e5", size = 225150, upload-time = "2025-04-03T19:34:18.625Z" }, - { url = "https://files.pythonhosted.org/packages/30/c8/2a534eb968abe372613a7f282d5c5bf921fed803499a199631b05c33e0b1/coreforecast-0.0.16-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ae259b7b8c22a9be25dbfae30fff48a984ba47e7bb171228a09ab1e68dae87c", size = 263682, upload-time = "2025-04-03T19:34:20.265Z" }, - { url = "https://files.pythonhosted.org/packages/13/70/e173ea405bbdb4dc2d6c7ed960d99631086abf5d343b641959b7056afec6/coreforecast-0.0.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57ca4f0e374fee7eddf3ab3c2be36e56df95a050f4fb8c28757ae3150980f06c", size = 287398, upload-time = "2025-04-03T19:34:21.879Z" }, - { url = "https://files.pythonhosted.org/packages/51/71/289cb5373cd26ea46eb371426010d005373ac92251c931be82c233e17052/coreforecast-0.0.16-cp312-cp312-win_amd64.whl", hash = "sha256:0b845827b0e00b017abe3c38e77f628cdca9770fd7bb02861f6edaed0d482d2b", size = 209584, upload-time = "2025-04-03T19:34:23.196Z" }, - { url = "https://files.pythonhosted.org/packages/0a/e3/2c75e0ca31a1e3c60ff2c4bb571fcd43adb69328c1511ea9bb3ea96e9278/coreforecast-0.0.16-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37fd79bf93e9336e9463c1c874da01e32ee2a4accea9bcee44f6e6494b0244f2", size = 259199, upload-time = "2025-04-03T19:34:26.897Z" }, - { url = "https://files.pythonhosted.org/packages/66/08/3fb9530dc6bdc8384b91f433dca31531b620341c9638665e596baa124b64/coreforecast-0.0.16-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:43f0a5317b8dcea5ac0866f1185643412a7cb99f8954f7e2d37716b1e73c2e1c", size = 225265, upload-time = "2025-04-03T19:34:28.207Z" }, - { url = "https://files.pythonhosted.org/packages/bc/0e/47e1f29fdc91129f2cc0c27e94426c4043841c16f7bfec69a1201d5af82e/coreforecast-0.0.16-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d250a62e9754b4fd9db07b0634428f4d42f971d50c18767dbd74e7f7e0d620d8", size = 263483, upload-time = "2025-04-03T19:34:29.398Z" }, - { url = "https://files.pythonhosted.org/packages/3c/43/258ef3207e51d6274aa2bbd128800306287c403cad4109a3b3cb7065d3cf/coreforecast-0.0.16-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b44b895f50909d7807a03d0f1941004452b897eb1719e934062a73108d700f20", size = 285407, upload-time = "2025-04-03T19:34:30.613Z" }, - { url = "https://files.pythonhosted.org/packages/3e/8f/bf51670b89377a6b34c324a787a63dfa102941417b7eb99a96447974700a/coreforecast-0.0.16-cp313-cp313-win_amd64.whl", hash = "sha256:3a688e1504f8805f329882047a45c1f0766a61c1edcf92fd90de4aa09a842c63", size = 209578, upload-time = "2025-04-03T19:34:32.401Z" }, -] - [[package]] name = "cycler" version = "0.12.1" @@ -414,7 +367,7 @@ wheels = [ [[package]] name = "darts" -version = "0.35.0" +version = "0.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "holidays" }, @@ -425,23 +378,18 @@ dependencies = [ { name = "numpy" }, { name = "pandas" }, { name = "pyod" }, - { name = "pytorch-lightning" }, { name = "requests" }, { name = "scikit-learn" }, { name = "scipy" }, { name = "shap" }, - { name = "statsforecast" }, { name = "statsmodels" }, - { name = "tensorboardx" }, - { name = "torch" }, { name = "tqdm" }, { name = "typing-extensions" }, { name = "xarray" }, - { name = "xgboost" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2d/91/6248a75687f0fe33d65e5bd5be852c51353b8282adad429d0e1cbc0b801b/darts-0.35.0.tar.gz", hash = "sha256:8f1371b682470eb21f870ef625701f1f323e4aaaec4a7bb0b2640543c876a3bd", size = 879146, upload-time = "2025-04-18T15:15:31.275Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/16/28040fc8bc0e32ebf98b2657e2e63d0a4ad9ee5945682864601944319354/darts-0.43.0.tar.gz", hash = "sha256:439827c0f7a0aec1148e56722671cbb785cada722d31e7effac0a8264c6e3f8a", size = 653086, upload-time = "2026-03-23T10:03:05.589Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/76/8301d25d658593216b27c38f551b1c0f58fcb0fc7e128151f5444db1a235/darts-0.35.0-py3-none-any.whl", hash = "sha256:59f003a66a04e63792de779c2fa8b4589441b92a220ef340dd555fed21d8b62b", size = 1026998, upload-time = "2025-04-18T15:15:29.276Z" }, + { url = "https://files.pythonhosted.org/packages/81/f8/7c3519f0f30ecf2d96bb59aff24441c28d89cf00662795a2e28e5d9cd1de/darts-0.43.0-py3-none-any.whl", hash = "sha256:2454e3b8491f1f9b38cedae5599d22d82f7419f1a8a887c736850926f826f206", size = 760462, upload-time = "2026-03-23T10:03:03.698Z" }, ] [[package]] @@ -579,20 +527,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, ] -[[package]] -name = "fs" -version = "2.4.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "appdirs" }, - { name = "setuptools" }, - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/a9/af5bfd5a92592c16cdae5c04f68187a309be8a146b528eac3c6e30edbad2/fs-2.4.16.tar.gz", hash = "sha256:ae97c7d51213f4b70b6a958292530289090de3a7e15841e108fbe144f069d313", size = 187441, upload-time = "2022-05-02T09:25:54.22Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/5c/a3d95dc1ec6cdeb032d789b552ecc76effa3557ea9186e1566df6aac18df/fs-2.4.16-py2.py3-none-any.whl", hash = "sha256:660064febbccda264ae0b6bace80a8d1be9e089e0a5eb2427b7d517f9a91545c", size = 135261, upload-time = "2022-05-02T09:25:52.363Z" }, -] - [[package]] name = "fsspec" version = "2025.9.0" @@ -607,19 +541,6 @@ http = [ { name = "aiohttp" }, ] -[[package]] -name = "fugue" -version = "0.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "adagio" }, - { name = "triad" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/a1/eca331442c758f8a6f23792dd10a51fb827fad1204805d6c70f02a35ee00/fugue-0.9.1.tar.gz", hash = "sha256:fb0f9a4780147ac8438be96efc50593e2d771d1cbf528ac56d3bcecd39915b50", size = 224340, upload-time = "2024-06-14T17:03:44.688Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/38/46a0ef179f7279207a3263afeb8da4dd73f44d00b6cc999c96a39112d284/fugue-0.9.1-py3-none-any.whl", hash = "sha256:5b91e55e6f243af6e2b901dc37914d954d8f0231627b68007850879f8848a3a3", size = 278186, upload-time = "2024-06-14T17:03:41.959Z" }, -] - [[package]] name = "greenlet" version = "3.2.4" @@ -1425,7 +1346,6 @@ name = "nvidia-nccl-cu12" version = "2.27.3" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/7b/8354b784cf73b0ba51e566b4baba3ddd44fe8288a3d39ef1e06cd5417226/nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9ddf1a245abc36c550870f26d537a9b6087fb2e2e3d6e0ef03374c6fd19d984f", size = 322397768, upload-time = "2025-06-03T21:57:30.234Z" }, { url = "https://files.pythonhosted.org/packages/5c/5b/4e4fff7bad39adf89f735f2bc87248c81db71205b62bcc0d5ca5b606b3c3/nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039", size = 322364134, upload-time = "2025-06-03T21:58:04.013Z" }, ] @@ -1727,16 +1647,17 @@ wheels = [ [[package]] name = "protobuf" -version = "6.32.1" +version = "7.34.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/6b/a0e95cad1ad7cc3f2c6821fcab91671bd5b78bd42afb357bb4765f29bc41/protobuf-7.34.1.tar.gz", hash = "sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280", size = 454708, upload-time = "2026-03-20T17:34:47.036Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/98/645183ea03ab3995d29086b8bf4f7562ebd3d10c9a4b14ee3f20d47cfe50/protobuf-6.32.1-cp310-abi3-win32.whl", hash = "sha256:a8a32a84bc9f2aad712041b8b366190f71dde248926da517bde9e832e4412085", size = 424411, upload-time = "2025-09-11T21:38:27.427Z" }, - { url = "https://files.pythonhosted.org/packages/8c/f3/6f58f841f6ebafe076cebeae33fc336e900619d34b1c93e4b5c97a81fdfa/protobuf-6.32.1-cp310-abi3-win_amd64.whl", hash = "sha256:b00a7d8c25fa471f16bc8153d0e53d6c9e827f0953f3c09aaa4331c718cae5e1", size = 435738, upload-time = "2025-09-11T21:38:30.959Z" }, - { url = "https://files.pythonhosted.org/packages/10/56/a8a3f4e7190837139e68c7002ec749190a163af3e330f65d90309145a210/protobuf-6.32.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8c7e6eb619ffdf105ee4ab76af5a68b60a9d0f66da3ea12d1640e6d8dab7281", size = 426454, upload-time = "2025-09-11T21:38:34.076Z" }, - { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" }, - { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" }, - { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" }, + { url = "https://files.pythonhosted.org/packages/ec/11/3325d41e6ee15bf1125654301211247b042563bcc898784351252549a8ad/protobuf-7.34.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7", size = 429247, upload-time = "2026-03-20T17:34:37.024Z" }, + { url = "https://files.pythonhosted.org/packages/eb/9d/aa69df2724ff63efa6f72307b483ce0827f4347cc6d6df24b59e26659fef/protobuf-7.34.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b", size = 325753, upload-time = "2026-03-20T17:34:38.751Z" }, + { url = "https://files.pythonhosted.org/packages/92/e8/d174c91fd48e50101943f042b09af9029064810b734e4160bbe282fa1caa/protobuf-7.34.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a", size = 340198, upload-time = "2026-03-20T17:34:39.871Z" }, + { url = "https://files.pythonhosted.org/packages/53/1b/3b431694a4dc6d37b9f653f0c64b0a0d9ec074ee810710c0c3da21d67ba7/protobuf-7.34.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4", size = 324267, upload-time = "2026-03-20T17:34:41.1Z" }, + { url = "https://files.pythonhosted.org/packages/85/29/64de04a0ac142fb685fd09999bc3d337943fb386f3a0ec57f92fd8203f97/protobuf-7.34.1-cp310-abi3-win32.whl", hash = "sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a", size = 426628, upload-time = "2026-03-20T17:34:42.536Z" }, + { url = "https://files.pythonhosted.org/packages/4d/87/cb5e585192a22b8bd457df5a2c16a75ea0db9674c3a0a39fc9347d84e075/protobuf-7.34.1-cp310-abi3-win_amd64.whl", hash = "sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c", size = 437901, upload-time = "2026-03-20T17:34:44.112Z" }, + { url = "https://files.pythonhosted.org/packages/88/95/608f665226bca68b736b79e457fded9a2a38c4f4379a4a7614303d9db3bc/protobuf-7.34.1-py3-none-any.whl", hash = "sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11", size = 170715, upload-time = "2026-03-20T17:34:45.384Z" }, ] [[package]] @@ -1755,38 +1676,52 @@ wheels = [ [[package]] name = "pyarrow" -version = "21.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487, upload-time = "2025-07-18T00:57:31.761Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b", size = 31243234, upload-time = "2025-07-18T00:55:03.812Z" }, - { url = "https://files.pythonhosted.org/packages/ea/cc/3b51cb2db26fe535d14f74cab4c79b191ed9a8cd4cbba45e2379b5ca2746/pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10", size = 32714370, upload-time = "2025-07-18T00:55:07.495Z" }, - { url = "https://files.pythonhosted.org/packages/24/11/a4431f36d5ad7d83b87146f515c063e4d07ef0b7240876ddb885e6b44f2e/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e", size = 41135424, upload-time = "2025-07-18T00:55:11.461Z" }, - { url = "https://files.pythonhosted.org/packages/74/dc/035d54638fc5d2971cbf1e987ccd45f1091c83bcf747281cf6cc25e72c88/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569", size = 42823810, upload-time = "2025-07-18T00:55:16.301Z" }, - { url = "https://files.pythonhosted.org/packages/2e/3b/89fced102448a9e3e0d4dded1f37fa3ce4700f02cdb8665457fcc8015f5b/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e", size = 43391538, upload-time = "2025-07-18T00:55:23.82Z" }, - { url = "https://files.pythonhosted.org/packages/fb/bb/ea7f1bd08978d39debd3b23611c293f64a642557e8141c80635d501e6d53/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c", size = 45120056, upload-time = "2025-07-18T00:55:28.231Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0b/77ea0600009842b30ceebc3337639a7380cd946061b620ac1a2f3cb541e2/pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6", size = 26220568, upload-time = "2025-07-18T00:55:32.122Z" }, - { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305, upload-time = "2025-07-18T00:55:35.373Z" }, - { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264, upload-time = "2025-07-18T00:55:39.303Z" }, - { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099, upload-time = "2025-07-18T00:55:42.889Z" }, - { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529, upload-time = "2025-07-18T00:55:47.069Z" }, - { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883, upload-time = "2025-07-18T00:55:53.069Z" }, - { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802, upload-time = "2025-07-18T00:55:57.714Z" }, - { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175, upload-time = "2025-07-18T00:56:01.364Z" }, - { url = "https://files.pythonhosted.org/packages/16/ca/c7eaa8e62db8fb37ce942b1ea0c6d7abfe3786ca193957afa25e71b81b66/pyarrow-21.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e99310a4ebd4479bcd1964dff9e14af33746300cb014aa4a3781738ac63baf4a", size = 31154306, upload-time = "2025-07-18T00:56:04.42Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e8/e87d9e3b2489302b3a1aea709aaca4b781c5252fcb812a17ab6275a9a484/pyarrow-21.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d2fe8e7f3ce329a71b7ddd7498b3cfac0eeb200c2789bd840234f0dc271a8efe", size = 32680622, upload-time = "2025-07-18T00:56:07.505Z" }, - { url = "https://files.pythonhosted.org/packages/84/52/79095d73a742aa0aba370c7942b1b655f598069489ab387fe47261a849e1/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f522e5709379d72fb3da7785aa489ff0bb87448a9dc5a75f45763a795a089ebd", size = 41104094, upload-time = "2025-07-18T00:56:10.994Z" }, - { url = "https://files.pythonhosted.org/packages/89/4b/7782438b551dbb0468892a276b8c789b8bbdb25ea5c5eb27faadd753e037/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:69cbbdf0631396e9925e048cfa5bce4e8c3d3b41562bbd70c685a8eb53a91e61", size = 42825576, upload-time = "2025-07-18T00:56:15.569Z" }, - { url = "https://files.pythonhosted.org/packages/b3/62/0f29de6e0a1e33518dec92c65be0351d32d7ca351e51ec5f4f837a9aab91/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:731c7022587006b755d0bdb27626a1a3bb004bb56b11fb30d98b6c1b4718579d", size = 43368342, upload-time = "2025-07-18T00:56:19.531Z" }, - { url = "https://files.pythonhosted.org/packages/90/c7/0fa1f3f29cf75f339768cc698c8ad4ddd2481c1742e9741459911c9ac477/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc56bc708f2d8ac71bd1dcb927e458c93cec10b98eb4120206a4091db7b67b99", size = 45131218, upload-time = "2025-07-18T00:56:23.347Z" }, - { url = "https://files.pythonhosted.org/packages/01/63/581f2076465e67b23bc5a37d4a2abff8362d389d29d8105832e82c9c811c/pyarrow-21.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:186aa00bca62139f75b7de8420f745f2af12941595bbbfa7ed3870ff63e25636", size = 26087551, upload-time = "2025-07-18T00:56:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ab/357d0d9648bb8241ee7348e564f2479d206ebe6e1c47ac5027c2e31ecd39/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:a7a102574faa3f421141a64c10216e078df467ab9576684d5cd696952546e2da", size = 31290064, upload-time = "2025-07-18T00:56:30.214Z" }, - { url = "https://files.pythonhosted.org/packages/3f/8a/5685d62a990e4cac2043fc76b4661bf38d06efed55cf45a334b455bd2759/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:1e005378c4a2c6db3ada3ad4c217b381f6c886f0a80d6a316fe586b90f77efd7", size = 32727837, upload-time = "2025-07-18T00:56:33.935Z" }, - { url = "https://files.pythonhosted.org/packages/fc/de/c0828ee09525c2bafefd3e736a248ebe764d07d0fd762d4f0929dbc516c9/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:65f8e85f79031449ec8706b74504a316805217b35b6099155dd7e227eef0d4b6", size = 41014158, upload-time = "2025-07-18T00:56:37.528Z" }, - { url = "https://files.pythonhosted.org/packages/6e/26/a2865c420c50b7a3748320b614f3484bfcde8347b2639b2b903b21ce6a72/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3a81486adc665c7eb1a2bde0224cfca6ceaba344a82a971ef059678417880eb8", size = 42667885, upload-time = "2025-07-18T00:56:41.483Z" }, - { url = "https://files.pythonhosted.org/packages/0a/f9/4ee798dc902533159250fb4321267730bc0a107d8c6889e07c3add4fe3a5/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fc0d2f88b81dcf3ccf9a6ae17f89183762c8a94a5bdcfa09e05cfe413acf0503", size = 43276625, upload-time = "2025-07-18T00:56:48.002Z" }, - { url = "https://files.pythonhosted.org/packages/5a/da/e02544d6997037a4b0d22d8e5f66bc9315c3671371a8b18c79ade1cefe14/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6299449adf89df38537837487a4f8d3bd91ec94354fdd2a7d30bc11c48ef6e79", size = 44951890, upload-time = "2025-07-18T00:56:52.568Z" }, - { url = "https://files.pythonhosted.org/packages/e5/4e/519c1bc1876625fe6b71e9a28287c43ec2f20f73c658b9ae1d485c0c206e/pyarrow-21.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:222c39e2c70113543982c6b34f3077962b44fca38c0bd9e68bb6781534425c10", size = 26371006, upload-time = "2025-07-18T00:56:56.379Z" }, +version = "23.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" }, + { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" }, + { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" }, + { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" }, + { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" }, + { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" }, + { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" }, + { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" }, + { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" }, + { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" }, + { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" }, + { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" }, + { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" }, + { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" }, + { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" }, + { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" }, + { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" }, + { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" }, + { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" }, + { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" }, + { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" }, + { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" }, + { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" }, + { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, ] [[package]] @@ -2392,9 +2327,11 @@ dependencies = [ { name = "pandas" }, { name = "pe-oudin" }, { name = "psutil" }, + { name = "pyarrow" }, { name = "scikit-learn" }, { name = "seaborn" }, { name = "suntime" }, + { name = "tensorboardx" }, { name = "tqdm" }, ] @@ -2408,7 +2345,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "click", specifier = ">=8.0" }, - { name = "darts", specifier = "==0.35.0" }, + { name = "darts", specifier = "==0.43.0" }, { name = "joblib", specifier = ">=1.5.2" }, { name = "lightning", specifier = "==2.5.1" }, { name = "mambapy", specifier = "==1.2.0" }, @@ -2418,9 +2355,11 @@ requires-dist = [ { name = "pandas", specifier = ">=2.3.3" }, { name = "pe-oudin", specifier = "==0.3" }, { name = "psutil", specifier = "==5.9.8" }, + { name = "pyarrow", specifier = ">=23.0.1" }, { name = "scikit-learn", specifier = ">=1.7.2" }, { name = "seaborn", specifier = ">=0.13.2" }, { name = "suntime", specifier = "==1.3.2" }, + { name = "tensorboardx", specifier = ">=2.6.4" }, { name = "tqdm", specifier = ">=4.67.1" }, ] @@ -2431,37 +2370,6 @@ dev = [ { name = "ruff", specifier = ">=0.13.2" }, ] -[[package]] -name = "statsforecast" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cloudpickle" }, - { name = "coreforecast" }, - { name = "fugue" }, - { name = "numba" }, - { name = "numpy" }, - { name = "pandas" }, - { name = "scipy" }, - { name = "statsmodels" }, - { name = "threadpoolctl" }, - { name = "tqdm" }, - { name = "utilsforecast" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/5b/7fbf787cc947ff358d3a4ec81911f0386aca66d643fdd5128f599a901479/statsforecast-2.0.1.tar.gz", hash = "sha256:dd856ad584f4d561b233da0db2b8e52b3a5cfa27109b0e0cb780293a836ad0b0", size = 2880305, upload-time = "2025-02-18T19:42:44.116Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/ae/62457c888696a723f7cc9de1d02b7ced1d8179d722d6261f7547de0bdbf9/statsforecast-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f26cb574db38f3ad5f2bc0a97126aa5b0b6b831e871bfb83dec352233737dd6a", size = 313006, upload-time = "2025-02-18T19:42:13.806Z" }, - { url = "https://files.pythonhosted.org/packages/39/b6/a49b13415f0b54f65f384a9d45354912592f11724331e40068fd8b19d3c8/statsforecast-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ec84068d8df8949ed499e73d0967daca023bcb5b385ad741a038f21f49b81f68", size = 299168, upload-time = "2025-02-18T19:42:15.29Z" }, - { url = "https://files.pythonhosted.org/packages/13/ce/60169b3b576f984cf1b51ec5ed623bd209d19ca79be85a1e67021ca7983e/statsforecast-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:826f3549d1f2c0ef76c7976619381f5a1f1dc504b2fbfbccf9f88e89362cb125", size = 340780, upload-time = "2025-02-18T19:42:18.254Z" }, - { url = "https://files.pythonhosted.org/packages/40/af/4ea162487e03f722d6d277ff409a645e9528293d8e5ca543b3a12fa2034c/statsforecast-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8855e5a950ff45c53da13c5febbfb47d6b946f8ca34d70b052e2ce7942263731", size = 354411, upload-time = "2025-02-18T19:42:20.435Z" }, - { url = "https://files.pythonhosted.org/packages/59/79/93136f585dc9953ea3895d5954df69ba6a722beae4cf02ce00a1b6281d9c/statsforecast-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:d6198030eeb48c02ebe43075830a638155e96ef748b106f9bd8703fe6951e11d", size = 276466, upload-time = "2025-02-18T19:42:21.792Z" }, - { url = "https://files.pythonhosted.org/packages/5e/98/4b99d0638fa3bd1ebb7c5aec452bc8cc25e83588d6b850f5557a850376cd/statsforecast-2.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7d3c5a41116e42a657510db905185b9faff152bf099a059f5aac4acaca4e5d39", size = 313855, upload-time = "2025-02-18T19:42:23.942Z" }, - { url = "https://files.pythonhosted.org/packages/b5/5f/5b3c02b27c37cc0085285a431c2c5c4893bb4c209516b7921360a28cfa70/statsforecast-2.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82ebc9cc5cb9fcad2a92adf7f248e509f5ac9b1402be0e35b50ffa1bcd4b74e2", size = 298730, upload-time = "2025-02-18T19:42:26.272Z" }, - { url = "https://files.pythonhosted.org/packages/a4/de/fe2fd9301101a2a96578295ae0b713d58c04161a7a68f0daf8aab40d1df9/statsforecast-2.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b8d6dc9f832658e4e17ee2a2d4d9553ce3c8b22766536fe28de9928ed8c0c64", size = 338046, upload-time = "2025-02-18T19:42:28.885Z" }, - { url = "https://files.pythonhosted.org/packages/c5/c2/a0819c3f4f618854a266e01c2316d247466a1d577809bc590657beb65bd3/statsforecast-2.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a191eab019140946f267a872fdeae8f61a3b27cfd69e7150e93c880ef90f75f", size = 353304, upload-time = "2025-02-18T19:42:31.054Z" }, - { url = "https://files.pythonhosted.org/packages/db/8c/7d8575a1cc09e739a4f66b349a4ef80f7a53e4eb18ae8b290315a6bb7eb6/statsforecast-2.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:61e0526cba8f5999268a6a6a43740d1e0f9923bb1ff7f54cfac90119bff08883", size = 277229, upload-time = "2025-02-18T19:42:32.465Z" }, -] - [[package]] name = "statsmodels" version = "0.14.5" @@ -2527,16 +2435,16 @@ wheels = [ [[package]] name = "tensorboardx" -version = "2.6.2.2" +version = "2.6.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "packaging" }, { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/9b/c2b5aba53f5e27ffcf249fc38485836119638f97d20b978664b15f97c8a6/tensorboardX-2.6.2.2.tar.gz", hash = "sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666", size = 4778030, upload-time = "2023-08-20T13:38:20.658Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/c5/d4cc6e293fb837aaf9f76dd7745476aeba8ef7ef5146c3b3f9ee375fe7a5/tensorboardx-2.6.4.tar.gz", hash = "sha256:b163ccb7798b31100b9f5fa4d6bc22dad362d7065c2f24b51e50731adde86828", size = 4769801, upload-time = "2025-06-10T22:37:07.419Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/44/71/f3e7c9b2ab67e28c572ab4e9d5fa3499e0d252650f96d8a3a03e26677f53/tensorboardX-2.6.2.2-py2.py3-none-any.whl", hash = "sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8", size = 101700, upload-time = "2023-08-20T13:38:18.089Z" }, + { url = "https://files.pythonhosted.org/packages/e0/1d/b5d63f1a6b824282b57f7b581810d20b7a28ca951f2d5b59f1eb0782c12b/tensorboardx-2.6.4-py3-none-any.whl", hash = "sha256:5970cf3a1f0a6a6e8b180ccf46f3fe832b8a25a70b86e5a237048a7c0beb18e2", size = 87201, upload-time = "2025-06-10T22:37:05.44Z" }, ] [[package]] @@ -2631,23 +2539,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, ] -[[package]] -name = "triad" -version = "0.9.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fs" }, - { name = "fsspec" }, - { name = "numpy" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/28/fca2981080bfb44e317b3fc6cc4119a0abf14f18e707a612764fcad28790/triad-0.9.8.tar.gz", hash = "sha256:5b67673124891981daf8afbab44b2e6358932ca35ef3ff38a25bc3e0f6f03f17", size = 56086, upload-time = "2024-06-28T06:11:32.537Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/c6/4aedce0522bb3c72f2d770e7e4c18b0e1f7716d2c70a865e94c89ebcf7e6/triad-0.9.8-py3-none-any.whl", hash = "sha256:2c0ba7d83977c6d4e7b59e3cc70727f858014ef7676c62d184aa8e63f7bef5de", size = 62340, upload-time = "2024-06-28T06:11:30.764Z" }, -] - [[package]] name = "triton" version = "3.4.0" @@ -2701,20 +2592,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, ] -[[package]] -name = "utilsforecast" -version = "0.2.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/d9/21e43a7419f0356043b52f72cd0262dd497f087fca78e90aebf8201a2339/utilsforecast-0.2.12.tar.gz", hash = "sha256:73f9dfd836a721a95c349f784bd75e18a4cb7c1469800e325414e22901e9775b", size = 41524, upload-time = "2025-02-24T19:41:56.25Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/75/9b/f6336ce71f4e6ed32877309314f549192cd6b982ad6d96fd8b1b5a230870/utilsforecast-0.2.12-py3-none-any.whl", hash = "sha256:acfba80bbf44e18433c206194f3ddd89cc28ff03aa0ba744b8040795a40b7b3f", size = 42219, upload-time = "2025-02-24T19:41:53.795Z" }, -] - [[package]] name = "wcmatch" version = "10.1" @@ -2750,26 +2627,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/a7/6eeb32e705d510a672f74135f538ad27f87f3d600845bfd3834ea3a77c7e/xarray-2025.9.1-py3-none-any.whl", hash = "sha256:3e9708db0d7915c784ed6c227d81b398dca4957afe68d119481f8a448fc88c44", size = 1364411, upload-time = "2025-09-30T05:28:51.294Z" }, ] -[[package]] -name = "xgboost" -version = "3.0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine != 'aarch64' and sys_platform == 'linux'" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7d/c0/561b88dabe82f45555dd2b68abd5d79f787d3e383436a6a54453d5deeb3f/xgboost-3.0.5.tar.gz", hash = "sha256:1a57a0d64a06b596992b664fe17dd1f9782138c6e35ad8fb6355e19a359fa50f", size = 1159729, upload-time = "2025-09-05T09:18:59.3Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/df/bf2416bfc138b1d2fa261ecb7f612ca00545aae126aeb1dc314df3291d73/xgboost-3.0.5-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:f974028c7a0c3ae51ef87e2900405436ddafff46452f6f334f5cc295e921429b", size = 2249197, upload-time = "2025-09-05T09:19:35.726Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/2c3e8a141180171aae913825bf5a200f37d1f7c598268cd8b6a219fb0eab/xgboost-3.0.5-py3-none-macosx_12_0_arm64.whl", hash = "sha256:40c324c329bf74f44571cdafb7aa7435366309d76ec14b3e16874862aeb14351", size = 2025998, upload-time = "2025-09-05T09:19:59.21Z" }, - { url = "https://files.pythonhosted.org/packages/66/68/e0e8285282ba81858d74d699cfee9e562a2a3cc7975f0fbd068b83aac559/xgboost-3.0.5-py3-none-manylinux2014_aarch64.whl", hash = "sha256:c580c82500ad566d927581381550561300492c0bc9c143b2e5be208d16f093d1", size = 4841604, upload-time = "2025-09-05T09:25:51.836Z" }, - { url = "https://files.pythonhosted.org/packages/69/32/eb7e862179194c6440eab63f834a3de064d6340a8b873b5520ac035891db/xgboost-3.0.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:d7f57a04629b52bae91a80e6721b9cdd009b605827a9eca67953675292b4487e", size = 4906211, upload-time = "2025-09-05T09:26:40.229Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d5/6c60111482f41fd680eb0e81e016498bea313cc00a6a4a41b38c8b45bd5c/xgboost-3.0.5-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:d0fe44aaca76e9c4598d3be98ae94661aa53e4c4ceb161dc7183258f0e6fc138", size = 4602832, upload-time = "2025-09-05T09:27:29.547Z" }, - { url = "https://files.pythonhosted.org/packages/64/ad/61a86228e981b15361ff963e84648b1a29ab43debd95f7c2b3ef9d94dca1/xgboost-3.0.5-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:a03210a3e54c9e543f480db9636fee57247cfcd1ae850b353aeac59eea5ca350", size = 94872848, upload-time = "2025-09-05T09:40:26.786Z" }, - { url = "https://files.pythonhosted.org/packages/00/5a/f43bad68b31269a72bdd66102732ea4473e98f421ee9f71379e35dcb56f5/xgboost-3.0.5-py3-none-win_amd64.whl", hash = "sha256:660774249d28a729ba8d22dd3d2c048c56e58f65a683b25ef3252e3383fe956f", size = 56826727, upload-time = "2025-09-05T09:23:55.462Z" }, -] - [[package]] name = "yarl" version = "1.20.1" From 2a2dbe360f104136ae1321a8980d5e927ef434aa Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Wed, 15 Apr 2026 10:45:45 +0200 Subject: [PATCH 02/20] SAPPHIRE PREDICTOR WORKS --- .bumpversion.toml | 2 +- .gitignore | 1 + examples/lamah_workflow.py | 6 +- notebooks/evaluate_runs_same_data.ipynb | 9 +- pyproject.toml | 8 +- scripts/compare_runs.py | 1 - scripts/explain_tft.py | 4 +- scripts/sensitivity_analysis.py | 17 +- scripts/watershed_visu.py | 354 ++++++++++++++ st_forecast/__init__.py | 2 +- st_forecast/cli/predict.py | 6 +- st_forecast/cli/train.py | 6 +- st_forecast/cli/tune.py | 4 +- st_forecast/custom_models/FANMixer.py | 2 - st_forecast/custom_models/lstm/model.py | 4 +- st_forecast/custom_models/lstm/module.py | 27 +- st_forecast/data_utils/__init__.py | 8 + st_forecast/data_utils/preparation.py | 246 +++++++++- st_forecast/data_utils/processors.py | 2 +- st_forecast/data_utils/scalers.py | 1 - st_forecast/evaluate_model.py | 18 +- st_forecast/load_data_darts.py | 7 +- st_forecast/operational/base_predictor.py | 7 +- st_forecast/operational/sapphire_predictor.py | 454 +++++++++--------- st_forecast/optimize.py | 17 - st_forecast/pipeline/artifacts.py | 2 + st_forecast/pipeline/forecast.py | 4 +- st_forecast/pipeline/hindcast.py | 4 +- st_forecast/pipeline/inference.py | 4 +- st_forecast/pipeline/workflow.py | 6 +- st_forecast/train_model.py | 12 +- st_forecast/transfer_fine_tuning.py | 17 +- st_forecast/utils/preprocessing_darts.py | 3 - st_forecast/visualization/_helpers.py | 2 +- st_forecast/visualization/_styles.py | 2 +- st_forecast/visualization/compare_runs.py | 13 +- .../test_probabilistic_metrics.py | 2 - tests/test_imports.py | 2 - tests/test_inference.py | 2 +- tests/test_preparation.py | 158 +++++- tests/test_region_specific_utils.py | 2 - uv.lock | 453 ++++++++++++++++- 42 files changed, 1506 insertions(+), 395 deletions(-) create mode 100644 scripts/watershed_visu.py diff --git a/.bumpversion.toml b/.bumpversion.toml index 5bb7801..5168549 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.1.6" +current_version = "0.1.9" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" diff --git a/.gitignore b/.gitignore index 98f085a..a8ac20b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ __pycache__/ /old_configs /st_forecast/data_utils/__pycache__ /comparison_output +/sapphire_ref diff --git a/examples/lamah_workflow.py b/examples/lamah_workflow.py index 6507250..c6afe05 100644 --- a/examples/lamah_workflow.py +++ b/examples/lamah_workflow.py @@ -37,7 +37,7 @@ ) from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, prepare_train_data, ) from st_forecast.evaluate_model import calculate_metrics @@ -191,7 +191,9 @@ def run_training( scalers_dict = train_result["scalers"] # Prepare validation data - val_result = prepare_inference_data(val_df, static_df, scalers_dict, prepare_config) + val_result = prepare_validation_data( + val_df, static_df, scalers_dict, prepare_config + ) # Extract components target_train, past_cov_train, future_cov_train, _ = extract_timeseries_components( diff --git a/notebooks/evaluate_runs_same_data.ipynb b/notebooks/evaluate_runs_same_data.ipynb index 6cbdc59..cd73c60 100644 --- a/notebooks/evaluate_runs_same_data.ipynb +++ b/notebooks/evaluate_runs_same_data.ipynb @@ -16,7 +16,6 @@ "outputs": [], "source": [ "import os\n", - "import sys\n", "import json\n", "import pandas as pd\n", "import numpy as np\n", @@ -25,14 +24,8 @@ "\n", "# import load_data_darts\n", "# adjust st_forecast module path\n", - "from st_forecast import get_data\n", - "from st_forecast import model_helper\n", "from st_forecast import evaluate_model\n", - "from st_forecast import visualization\n", - "from st_forecast.data_utils.loaders import load_data_df\n", - "from st_forecast.custom_models.FANMixer import FANMixerModel\n", - "from st_forecast.custom_models.ExoMamba import ExoMambaModel\n", - "from st_forecast.custom_models.ExoLSTM import ExoLSTMModel" + "from st_forecast.data_utils.loaders import load_data_df" ] }, { diff --git a/pyproject.toml b/pyproject.toml index 4f94605..259cee6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "st-forecast" -version = "0.1.8" +version = "0.1.9" description = "Add your description here" readme = "README.md" -requires-python = ">=3.11.9" +requires-python = ">=3.11" dependencies = [ "click>=8.0", "darts==0.43.0", @@ -30,6 +30,10 @@ st-forecast = "st_forecast.cli:main" [dependency-groups] dev = [ "bump-my-version>=1.2.7", + "cartopy>=0.25.0", + "contextily>=1.7.0", + "geopandas>=1.1.3", + "matplotlib-scalebar>=0.9.0", "pytest>=8.4.2", "ruff>=0.13.2", ] diff --git a/scripts/compare_runs.py b/scripts/compare_runs.py index e393260..3b54e8f 100644 --- a/scripts/compare_runs.py +++ b/scripts/compare_runs.py @@ -48,7 +48,6 @@ "TiDE", "TFT", "Ensemble", - ] OUTPUT_DIR: Path = Path("comparison_output/kgz") diff --git a/scripts/explain_tft.py b/scripts/explain_tft.py index 5e539ba..12ca219 100644 --- a/scripts/explain_tft.py +++ b/scripts/explain_tft.py @@ -20,7 +20,7 @@ from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, ) from st_forecast.pipeline.artifacts import load_run_artifacts from st_forecast.pipeline.inference import load_model @@ -102,7 +102,7 @@ def _load_data( "era5": scalers.era5, "static": scalers.static, } - data_result = prepare_inference_data( + data_result = prepare_validation_data( temporal_df, static_df, scalers_dict, prepare_config ) target, past_cov, future_cov, basin_codes = extract_timeseries_components( diff --git a/scripts/sensitivity_analysis.py b/scripts/sensitivity_analysis.py index 9844187..1cb518f 100644 --- a/scripts/sensitivity_analysis.py +++ b/scripts/sensitivity_analysis.py @@ -28,7 +28,7 @@ from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, ) from st_forecast.pipeline.artifacts import RunConfig, Scalers, load_run_artifacts from st_forecast.pipeline.inference import ( @@ -42,7 +42,6 @@ logger = logging.getLogger(__name__) - def _build_collector(config: RunConfig) -> RawDataCollector | CaravanDataCollector: if getattr(config, "data_source", "legacy") == "caravan" and config.caravan: caravan_config = CaravanConfig( @@ -386,7 +385,9 @@ def _seeded_forecast(**kwargs: ...) -> pd.DataFrame: def plot_ablation_boxplot(results_df: pd.DataFrame, output_dir: Path) -> None: """Boxplot for ablation experiments: each point = one basin's mean error.""" - basin_means = results_df.groupby(["code", "ablation"])["pct_error"].mean().reset_index() + basin_means = ( + results_df.groupby(["code", "ablation"])["pct_error"].mean().reset_index() + ) ablations = sorted(basin_means["ablation"].unique()) fig, ax = plt.subplots(figsize=figsize(0.7, 0.5)) @@ -423,9 +424,9 @@ def plot_sensitivity_boxplot(results_df: pd.DataFrame, output_dir: Path) -> None .mean() .reset_index() ) - basin_means["noise_label"] = ( - (basin_means["noise_level"] * 100).astype(int).astype(str) + "%" - ) + basin_means["noise_label"] = (basin_means["noise_level"] * 100).astype(int).astype( + str + ) + "%" noise_labels = sorted( basin_means["noise_label"].unique(), @@ -494,7 +495,7 @@ def main() -> None: "--noise-levels", type=float, nargs="+", - default=[ 0.05, 0.10, 0.15, 0.20, 0.30 , 0.50], + default=[0.05, 0.10, 0.15, 0.20, 0.30, 0.50], help="Noise levels as fractions (default: 0.05 0.10 0.15 0.20, 0.30, 0.50 for 5% to 50% noise)", ) parser.add_argument( @@ -580,7 +581,7 @@ def main() -> None: "era5": scalers.era5, "static": scalers.static, } - data_result = prepare_inference_data( + data_result = prepare_validation_data( temporal_df, static_df, scalers_dict, prepare_config ) target_list, past_cov_list, future_cov_list, basin_codes = ( diff --git a/scripts/watershed_visu.py b/scripts/watershed_visu.py new file mode 100644 index 0000000..8b9f23d --- /dev/null +++ b/scripts/watershed_visu.py @@ -0,0 +1,354 @@ +""" +Kyrgyzstan Watershed Map – Hillshade terrain with Spectral-colored basins by region. +Matches the visual style of the R/tmap `nice_viz.R` reference script. + +Requirements: + geopandas matplotlib cartopy matplotlib-scalebar rasterio shapely numpy +""" + +import geopandas as gpd +import matplotlib.patches as mpatches +import matplotlib.patheffects as pe +import matplotlib.pyplot as plt +import numpy as np +import rasterio +from matplotlib_scalebar.scalebar import ScaleBar +from rasterio.windows import from_bounds +import cartopy.crs as ccrs +import cartopy.feature as cfeature +from cartopy.io.shapereader import Reader, natural_earth + +# ── 0. CONFIG ──────────────────────────────────────────────────────────────── +SHAPEFILE = ( + "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/" + "SAPPHIRE_Central_Asia_Technical_Work/GIS/HRU_Gateway/00003/" + "HRU_KRG_ML_MODEL_BASINS_1d.shp" +) + +VIZ_BASE = ( + "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/" + "REFERENCE_CaseStudyPacks/CENTRAL_ASIA_DOMAIN/VISUALIZATION" +) + +HILLSHADE_PATH = f"{VIZ_BASE}/raster/hill_ca.tif" +RIVERS_PATHS = { + 1: f"{VIZ_BASE}/shapes/wmobb_rivnets_Q09_10_ca_large_river_basins.shp", + 2: f"{VIZ_BASE}/shapes/wmobb_rivnets_Q08_09_ca_large_river_basins.shp", + 3: f"{VIZ_BASE}/shapes/wmobb_rivnets_Q07_08_ca_large_river_basins.shp", +} +COUNTRIES_PATH = f"{VIZ_BASE}/shapes/GADM36_Central_Asia_incl_IRAN.shp" +LAKES_PATH = f"{VIZ_BASE}/shapes/HydroLAKES_polys_v10_CA_large_basins.shp" +BASINS_COMPLETE_PATH = f"{VIZ_BASE}/shapes/basins_complete_gpkg.gpkg" + +CRS_GEO = "EPSG:4326" +OUTPUT_DPI = 200 +OUTPUT_FILE = "kyrgyzstan_watersheds.png" +LAKE_AREA_CUTOFF = 500 # km2 + +# ── Colours (matching R script) ────────────────────────────────────────────── +C_LAKE = "#c6dbef" +C_BORDER = "darkgrey" +C_HIGHLIGHT = "#E63946" + +RIVER_STYLES = { + 1: {"color": "darkblue", "linewidth": 1.0, "alpha": 0.5}, + 2: {"color": "blue", "linewidth": 0.666, "alpha": 0.5}, + 3: {"color": "#4169e1", "linewidth": 0.333, "alpha": 0.5}, +} + +REGION_RENAME = { + "AMU_DARYA": "Amu Darya", + "SYR_DARYA": "Syr Darya", + "CHU_TALAS": "Chu-Talas", + "ISSYKUL": "Issyk-Kul", + "MURGHAB_HARIRUD": "Murghab-Harirud", +} + +CNT_LABELS = [ + ("CHN", 78, 39), + ("KGZ", 74.5, 41.5), + ("KAZ", 72.5, 43.3), + ("UZB", 64, 41), + ("TKM", 62, 38), + ("TJK", 68.5, 38), + ("AFG", 69, 34.5), +] + +# ── 1. LOAD DATA ───────────────────────────────────────────────────────────── +# Watersheds +ws = gpd.read_file(SHAPEFILE).to_crs(CRS_GEO) + +# Join REGION from basins_complete +bc = gpd.read_file(BASINS_COMPLETE_PATH, columns=["CODE", "REGION"]) +code_to_region = dict(zip(bc["CODE"].astype(str), bc["REGION"])) +ws["REGION"] = ws["CODE"].astype(str).map(code_to_region) +ws.loc[ws["REGION"].isna(), "REGION"] = "SYR_DARYA" # CODE 16936 +ws["REGION"] = ws["REGION"].map(REGION_RENAME) + +# Build Spectral palette for regions present +regions_present = sorted(ws["REGION"].unique()) +spectral = plt.colormaps["Spectral"] +n = max(len(regions_present) - 1, 1) +region_colors = {r: spectral(i / n) for i, r in enumerate(regions_present)} + +# Countries +countries = gpd.read_file(COUNTRIES_PATH).to_crs(CRS_GEO) +kyrgyzstan = countries[countries["NAME_0"].isin(["Kyrgyzstan", "Kyrgyz Republic"])] + +# Lakes (reproject from UTM 42N, filter large) +lakes = gpd.read_file(LAKES_PATH).to_crs(CRS_GEO) +lakes = lakes[lakes["Lake_area"] > LAKE_AREA_CUTOFF] + +# Rivers (3 tiers, make valid) +rivers = {} +for tier, path in RIVERS_PATHS.items(): + gdf = gpd.read_file(path).to_crs(CRS_GEO) + gdf["geometry"] = gdf.geometry.make_valid() + rivers[tier] = gdf + +# ── 2. MAP EXTENT ───────────────────────────────────────────────────────────── +b = ws.total_bounds # [W, S, E, N] +pad_x = (b[2] - b[0]) * 0.15 +pad_y = (b[3] - b[1]) * 0.15 +xlim = (b[0] - pad_x, b[2] + pad_x) +ylim = (b[1] - pad_y, b[3] + pad_y) + +# ── 3. LOAD HILLSHADE (windowed) ───────────────────────────────────────────── +with rasterio.open(HILLSHADE_PATH) as src: + # Clamp map extent to hillshade coverage to avoid edge artifacts + hs_bounds = src.bounds + xlim = (max(xlim[0], hs_bounds.left), min(xlim[1], hs_bounds.right)) + ylim = (max(ylim[0], hs_bounds.bottom), min(ylim[1], hs_bounds.top)) + + window = from_bounds(xlim[0], ylim[0], xlim[1], ylim[1], src.transform) + hillshade = src.read(1, window=window) + nodata = src.nodata + hillshade = np.ma.masked_where( + hillshade < -1e30 if nodata is None else hillshade == nodata, hillshade + ) + win_transform = src.window_transform(window) + hs_extent = [ + win_transform.c, + win_transform.c + win_transform.a * hillshade.shape[1], + win_transform.f + win_transform.e * hillshade.shape[0], + win_transform.f, + ] + +# ── 4. FIGURE LAYOUT ───────────────────────────────────────────────────────── +fig = plt.figure(figsize=(14, 11), facecolor="white") +ax_main = fig.add_axes([0.02, 0.02, 0.96, 0.96]) +ax_inset = fig.add_axes([0.72, 0.09, 0.26, 0.30], projection=ccrs.PlateCarree()) + +# ── 5. MAIN MAP ────────────────────────────────────────────────────────────── +ax_main.set_xlim(xlim) +ax_main.set_ylim(ylim) +ax_main.set_aspect(1 / np.cos(np.radians(41))) +ax_main.axis("off") +ax_main.set_facecolor("white") +for sp in ax_main.spines.values(): + sp.set_visible(False) + +# z=0: Hillshade background +ax_main.imshow( + hillshade, + cmap="Greys_r", + extent=hs_extent, + aspect="auto", + zorder=0, + interpolation="bilinear", +) + +# z=0.5: Kyrgyzstan subtle gray shading +kyrgyzstan.plot( + ax=ax_main, facecolor="#d0d0d0", edgecolor="none", alpha=0.15, zorder=0.5 +) + +# z=1: Country borders +countries.boundary.plot(ax=ax_main, edgecolor=C_BORDER, linewidth=0.8, zorder=1) + +# z=2: Lakes +if not lakes.empty: + lakes.plot(ax=ax_main, facecolor=C_LAKE, edgecolor="none", alpha=1, zorder=2) + +# z=3: Basins colored by REGION +for region, color in region_colors.items(): + subset = ws[ws["REGION"] == region] + subset.plot( + ax=ax_main, + facecolor=(*color[:3], 0.35), + edgecolor="black", + linewidth=0.5, + zorder=3, + ) + +# z=3.5: Kyrgyzstan border (thick black) +kyrgyzstan.boundary.plot(ax=ax_main, edgecolor="black", linewidth=1.2, zorder=3.5) + +# z=4-6: Rivers (3 tiers) +for tier in [1, 2, 3]: + style = RIVER_STYLES[tier] + rivers[tier].plot( + ax=ax_main, + color=style["color"], + linewidth=style["linewidth"], + alpha=style["alpha"], + zorder=3 + tier, + ) + +# z=7: Country labels +for lbl, lon, lat in CNT_LABELS: + if xlim[0] < lon < xlim[1] and ylim[0] < lat < ylim[1]: + ax_main.text( + lon, + lat, + lbl, + fontsize=9, + fontweight="bold", + color="#333333", + ha="center", + va="center", + path_effects=[pe.withStroke(linewidth=2.5, foreground="white")], + zorder=7, + ) + +# Compass (north arrow, top-left) +ax_main.annotate( + "N", + xy=(0.035, 0.95), + xytext=(0.035, 0.90), + xycoords="axes fraction", + fontsize=14, + fontweight="bold", + color="#333333", + ha="center", + arrowprops={"arrowstyle": "-|>", "color": "#333333", "lw": 2.0}, + zorder=10, +) + +# Scale bar +ax_main.add_artist( + ScaleBar( + 1, + units="deg", + dimension="angle", + location="lower left", + rotation="horizontal-only", + color="#333333", + box_alpha=0.65, + box_color="white", + font_properties={"size": 9}, + ) +) + + +# ── 6. LEGEND (embedded in main axes, right-bottom) ────────────────────────── +legend_handles = [] +for region in regions_present: + c = region_colors[region] + legend_handles.append( + mpatches.Patch( + facecolor=(*c[:3], 0.35), edgecolor="black", lw=0.5, label=region + ) + ) + +leg = ax_main.legend( + handles=legend_handles, + loc="lower right", + bbox_to_anchor=(0.72, 0.0), + title="Basins", + title_fontsize=10, + fontsize=8.5, + frameon=True, + fancybox=False, + edgecolor="black", + facecolor="white", + framealpha=0.9, + handlelength=1.8, + handleheight=1.1, + borderpad=0.8, + labelspacing=0.6, +) +leg.get_frame().set_linewidth(0.8) + +# ── 7. INSET MAP ───────────────────────────────────────────────────────────── +ax_inset.set_extent([52, 90, 34, 56], crs=ccrs.PlateCarree()) +ax_inset.set_facecolor("#c9dff0") + +ax_inset.add_feature( + cfeature.NaturalEarthFeature( + "physical", "land", "50m", facecolor="#eeefec", zorder=0 + ) +) +ax_inset.add_feature( + cfeature.NaturalEarthFeature( + "physical", + "lakes", + "50m", + facecolor=C_LAKE, + edgecolor="steelblue", + linewidth=0.3, + zorder=1, + ) +) +ax_inset.add_feature( + cfeature.RIVERS.with_scale("50m"), + edgecolor="steelblue", + linewidth=0.5, + alpha=0.7, + zorder=2, +) +ax_inset.add_feature( + cfeature.BORDERS.with_scale("50m"), + edgecolor="#444444", + linewidth=0.5, + linestyle="--", + zorder=3, +) + +# Kyrgyzstan highlighted +for geom, rec in zip( + Reader(natural_earth("50m", "cultural", "admin_0_countries")).geometries(), + Reader(natural_earth("50m", "cultural", "admin_0_countries")).records(), +): + if rec.attributes["NAME"] in ("Kyrgyzstan", "Kyrgyz Republic"): + ax_inset.add_geometries( + [geom], + ccrs.PlateCarree(), + facecolor=C_HIGHLIGHT, + edgecolor="white", + linewidth=0.8, + alpha=0.80, + zorder=4, + ) + +# Country labels (inset) +for name, lon, lat in [ + ("Kazakhstan", 67.0, 48.5), + ("Uzbekistan", 62.5, 41.5), + ("Tajikistan", 71.5, 38.2), + ("China", 86.0, 43.5), + ("Kyrgyzstan", 74.5, 41.5), + ("Afghanistan", 66.0, 35.2), +]: + ax_inset.text( + lon, + lat, + name, + transform=ccrs.PlateCarree(), + fontsize=6, + color="#222222", + ha="center", + fontweight="bold", + path_effects=[pe.withStroke(linewidth=1.8, foreground="white")], + zorder=5, + ) + +ax_inset.set_title("Location", color="#222222", fontsize=9, fontweight="bold", pad=4) +for sp in ax_inset.spines.values(): + sp.set_edgecolor("#555555") + sp.set_linewidth(1.5) + +# ── 8. SAVE ────────────────────────────────────────────────────────────────── +plt.savefig(OUTPUT_FILE, dpi=OUTPUT_DPI, bbox_inches="tight", facecolor="white") +print(f"Saved -> {OUTPUT_FILE}") +plt.show() diff --git a/st_forecast/__init__.py b/st_forecast/__init__.py index 22b61c4..ea51657 100644 --- a/st_forecast/__init__.py +++ b/st_forecast/__init__.py @@ -7,7 +7,7 @@ from st_forecast.pipeline.artifacts import RunConfig from st_forecast.pipeline.workflow import TrainModelResult, train_model -__version__ = "0.1.8" +__version__ = "0.1.9" # Package-level constants BASIN_ID_COL = "code" diff --git a/st_forecast/cli/predict.py b/st_forecast/cli/predict.py index cfb2077..7cc759a 100644 --- a/st_forecast/cli/predict.py +++ b/st_forecast/cli/predict.py @@ -13,7 +13,7 @@ from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, ) from st_forecast.pipeline.artifacts import load_run_artifacts from st_forecast.pipeline.inference import ( @@ -176,7 +176,7 @@ def predict_cli( # Prepare config for data preparation prepare_config = run_config.to_prepare_config() - # Convert Scalers to dict format for prepare_inference_data + # Convert Scalers to dict format for prepare_validation_data scalers_dict = { "discharge": scalers.discharge, "era5": scalers.era5, @@ -184,7 +184,7 @@ def predict_cli( } # Prepare data using inference mode (apply pre-computed scalers) - data_result = prepare_inference_data( + data_result = prepare_validation_data( selected_temporal, static_df, scalers_dict, prepare_config ) timeseries = data_result["timeseries"] diff --git a/st_forecast/cli/train.py b/st_forecast/cli/train.py index c243ce9..15bf600 100644 --- a/st_forecast/cli/train.py +++ b/st_forecast/cli/train.py @@ -14,7 +14,7 @@ from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, prepare_train_data, ) from st_forecast.pipeline.artifacts import Scalers, load_run_artifacts @@ -195,7 +195,7 @@ def train_cli(run_folder: Path, config: Path | None, seed: int) -> None: "era5": existing_scalers.era5, "static": existing_scalers.static, } - train_result = prepare_inference_data( + train_result = prepare_validation_data( train_temporal, static_df, scalers_dict, prepare_config ) train_timeseries = train_result["timeseries"] @@ -203,7 +203,7 @@ def train_cli(run_folder: Path, config: Path | None, seed: int) -> None: # Prepare validation data if val_temporal is not None: - val_result = prepare_inference_data( + val_result = prepare_validation_data( val_temporal, static_df, scalers_dict, prepare_config ) val_timeseries = val_result["timeseries"] diff --git a/st_forecast/cli/tune.py b/st_forecast/cli/tune.py index 8d7250c..d62bfe9 100644 --- a/st_forecast/cli/tune.py +++ b/st_forecast/cli/tune.py @@ -16,7 +16,7 @@ from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, prepare_train_data, ) from st_forecast.pipeline.artifacts import RunConfig, Scalers @@ -149,7 +149,7 @@ def train_fn(trial_config: RunConfig, trial_dir: Path) -> TrainingResult: scalers.save(scalers_folder) if val_temporal is not None: - val_result = prepare_inference_data( + val_result = prepare_validation_data( val_temporal, static_df, scalers_dict, prepare_config ) val_timeseries = val_result["timeseries"] diff --git a/st_forecast/custom_models/FANMixer.py b/st_forecast/custom_models/FANMixer.py index 09090f0..a48f048 100644 --- a/st_forecast/custom_models/FANMixer.py +++ b/st_forecast/custom_models/FANMixer.py @@ -23,8 +23,6 @@ TimeBatchNorm2d, ACTIVATIONS, NORMS, - _ConditionalMixerLayer, - TSMixerModel, ) from darts.utils.torch import MonteCarloDropout diff --git a/st_forecast/custom_models/lstm/model.py b/st_forecast/custom_models/lstm/model.py index 69a65af..75b5b2f 100644 --- a/st_forecast/custom_models/lstm/model.py +++ b/st_forecast/custom_models/lstm/model.py @@ -55,7 +55,9 @@ def _create_model( output_dim = future_target.shape[1] past_cov_dim = past_covariates.shape[1] if past_covariates is not None else 0 - future_cov_dim = future_covariates.shape[1] if future_covariates is not None else 0 + future_cov_dim = ( + future_covariates.shape[1] if future_covariates is not None else 0 + ) if static_covariates is not None: if len(static_covariates.shape) > 2: diff --git a/st_forecast/custom_models/lstm/module.py b/st_forecast/custom_models/lstm/module.py index 49b649d..45e9608 100644 --- a/st_forecast/custom_models/lstm/module.py +++ b/st_forecast/custom_models/lstm/module.py @@ -103,7 +103,11 @@ def forward( x, x_future_covariates, x_static_covariates = x_in batch_size = x.shape[0] past_len = x.shape[1] - future_len = x_future_covariates.shape[1] if x_future_covariates is not None else self.output_chunk_length + future_len = ( + x_future_covariates.shape[1] + if x_future_covariates is not None + else self.output_chunk_length + ) total_len = past_len + future_len # Flatten static covariates @@ -134,7 +138,8 @@ def forward( # Random observation masking during training if self.training and self.obs_mask_prob > 0.0 and self.use_past_target: keep_mask = ( - torch.rand(batch_size, past_len, 1, device=x.device) >= self.obs_mask_prob + torch.rand(batch_size, past_len, 1, device=x.device) + >= self.obs_mask_prob ) past_features = torch.cat( [ @@ -146,8 +151,11 @@ def forward( # Zero-pad into future extended_past = torch.zeros( - batch_size, total_len, self.past_input_dim, - dtype=x.dtype, device=x.device, + batch_size, + total_len, + self.past_input_dim, + dtype=x.dtype, + device=x.device, ) extended_past[:, :past_len, :] = past_features parts.append(extended_past) @@ -157,14 +165,19 @@ def forward( indicator = torch.zeros( batch_size, total_len, 1, dtype=x.dtype, device=x.device ) - indicator[:, :past_len, :] = keep_mask.float() if keep_mask is not None else 1.0 + indicator[:, :past_len, :] = ( + keep_mask.float() if keep_mask is not None else 1.0 + ) parts.append(indicator) # Future covariates → zero-padded to [B, L+T, F] if x_future_covariates is not None and self.future_cov_dim > 0: padded_future = torch.zeros( - batch_size, total_len, self.future_cov_dim, - dtype=x.dtype, device=x.device, + batch_size, + total_len, + self.future_cov_dim, + dtype=x.dtype, + device=x.device, ) padded_future[:, past_len:, :] = x_future_covariates parts.append(padded_future) diff --git a/st_forecast/data_utils/__init__.py b/st_forecast/data_utils/__init__.py index 9b08e30..2a22721 100644 --- a/st_forecast/data_utils/__init__.py +++ b/st_forecast/data_utils/__init__.py @@ -33,8 +33,12 @@ add_derived_features, ) from .preparation import ( + adjust_feature_names, + apply_static_scalers, prepare_train_data, prepare_inference_data, + prepare_validation_data, + resolve_static_features, ) __all__ = [ @@ -67,6 +71,10 @@ "scale_static_features", "save_scalers", # Preparation (train/inference pipeline) + "adjust_feature_names", + "apply_static_scalers", "prepare_train_data", "prepare_inference_data", + "prepare_validation_data", + "resolve_static_features", ] diff --git a/st_forecast/data_utils/preparation.py b/st_forecast/data_utils/preparation.py index e60d59a..0c8821d 100644 --- a/st_forecast/data_utils/preparation.py +++ b/st_forecast/data_utils/preparation.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) -def _resolve_static_features( +def resolve_static_features( static_df: pd.DataFrame, static_features: list[str], static_id_col: str, @@ -73,18 +73,45 @@ def prepare_train_data( ) +def prepare_validation_data( + temporal_df: pd.DataFrame, + static_df: pd.DataFrame, + scalers: dict, + config: dict, +) -> dict: + """Prepare validation/test data: apply pre-computed scalers, return stacked timeseries. + + Use this for training pipeline val/test splits where all data (target + features) + is fully available. For operational forecasting where covariates extend beyond + the target, use prepare_inference_data instead. + + Returns + ------- + dict + {'timeseries': list[TimeSeries]} + """ + result = _build_timeseries_pipeline( + temporal_df, static_df, config, mode="transform", scalers=scalers + ) + return {"timeseries": result["timeseries"]} + + def prepare_inference_data( temporal_df: pd.DataFrame, static_df: pd.DataFrame, scalers: dict, config: dict, ) -> dict: - """Prepare inference data: apply pre-computed scalers. + """Prepare inference data with separate target, past, and future covariates. + + Handles the operational forecasting scenario where target (discharge) is + available up to time t, while covariates extend to t + future_horizon. Parameters ---------- temporal_df : pd.DataFrame - Combined temporal data with columns: date, code, discharge, and forcing features + Combined temporal data with columns: date, code, discharge, and forcing features. + Discharge may have NaN for future dates where only covariates are available. static_df : pd.DataFrame Static basin features indexed by CODE scalers : dict @@ -96,13 +123,206 @@ def prepare_inference_data( ------- dict { - 'timeseries': list[TimeSeries] + 'target': list[TimeSeries], # discharge up to last observation per basin + 'past_covariates': list[TimeSeries] | None, # features up to last observation + 'future_covariates': list[TimeSeries] | None, # future features for full range + 'codes': list[int] } """ - result = _build_timeseries_pipeline( - temporal_df, static_df, config, mode="transform", scalers=scalers + features = config.get("features", []) + exog_features = config.get("exog_features", []) + moving_windows = config.get("moving_windows", []) + shift_moving_avg = config.get("shift_moving_avg", False) + forecast_horizon = config.get("forecast_horizon", 6) + input_chunk_length = config.get("input_chunk_length", 60) + + scale_discharge = config.get("scale_discharge", True) + discharge_scaler_type = config.get("discharge_scaler_type", "standard") + global_scaling = config.get("global_scaling", False) + + scale_forcing = config.get("scale_forcing", True) + forcing_scaler_type = config.get("forcing_scaler_type", "minmax") + + scale_static = config.get("scale_static", True) + static_scaler_type = config.get("static_scaler_type", "minmax") + + past_features = config.get("past_features", []) + future_features = config.get("future_features", []) + + date_col = config.get("date_col", "date") + basin_id_col = config.get("basin_id_col", "code") + target_col = config.get("target_col", "discharge") + static_id_col = config.get("static_id_col", "CODE") + + # Validate target column exists (outer merge can create suffixed duplicates) + if target_col not in temporal_df.columns: + suffixed = [c for c in temporal_df.columns if c.startswith(f"{target_col}_")] + raise ValueError( + f"Target column '{target_col}' not found in temporal_df. " + f"Columns: {temporal_df.columns.tolist()}. " + f"Possible merge conflict — found suffixed columns: {suffixed}. " + f"Ensure past_measurements and covariates don't share the target column name." + ) + + logger.info( + f"prepare_inference_data: temporal_df shape={temporal_df.shape}, " + f"columns={temporal_df.columns.tolist()}, " + f"basins={temporal_df[basin_id_col].unique().tolist()}, " + f"non-null {target_col}={temporal_df[target_col].notna().sum()}" ) - return {"timeseries": result["timeseries"]} + + # Step 1: Split into discharge and forcing + discharge_df, forcing_df = _split_temporal_data( + temporal_df, date_col=date_col, basin_id_col=basin_id_col, target_col=target_col + ) + discharge_df = discharge_df.drop_duplicates( + subset=[basin_id_col, date_col], keep="last" + ) + forcing_df = forcing_df.drop_duplicates( + subset=[basin_id_col, date_col], keep="last" + ) + + # Step 2: Derived features + if "PET" not in forcing_df.columns or "daylight_hours" not in forcing_df.columns: + from .feature_engineers import add_derived_features + + forcing_df = add_derived_features( + forcing_df, static_df, date_col=date_col, basin_id_col=basin_id_col + ) + + # Step 3: Reindex — use full date range covering both discharge and forcing + min_date = min(discharge_df[date_col].min(), forcing_df[date_col].min()) + max_date = max(discharge_df[date_col].max(), forcing_df[date_col].max()) + discharge_df = reindex_dataframe( + discharge_df, min_date, max_date, date_col=date_col, basin_id_col=basin_id_col + ) + forcing_df = reindex_dataframe( + forcing_df, min_date, max_date, date_col=date_col, basin_id_col=basin_id_col + ) + + # Step 4: Unit transformations + transform_mm = config.get("transform_mm", False) + transform_log = config.get("transform_log", False) + discharge_df = transform_discharge_units( + discharge_df, static_df, transform_mm, transform_log + ) + + # Step 5: Moving averages + for window in moving_windows: + forcing_df = add_moving_avg( + discharge=discharge_df, + era5=forcing_df, + shifted_bool=shift_moving_avg, + shift=forecast_horizon, + moving_window=window, + ) + if shift_moving_avg: + col_mapping = { + "shifted_moving_average_discharge": f"moving_avr_dis_{window}_shifted" + } + else: + col_mapping = {"moving_average_discharge": f"moving_avr_dis_{window}"} + forcing_df = forcing_df.rename(columns=col_mapping) + + # Step 6: Scale discharge + if scale_discharge: + discharge_df = apply_discharge_scalers( + discharge_df, scalers["discharge"], discharge_scaler_type, global_scaling + ) + + # Step 7: Scale forcing + if scale_forcing and exog_features: + forcing_df = apply_era5_scalers( + forcing_df, scalers["era5"], exog_features, forcing_scaler_type + ) + + # Step 8: Scale static features + if scale_static: + static_features_config = config.get("static_features", []) + static_to_scale = resolve_static_features( + static_df, static_features_config, static_id_col + ) + static_features = apply_static_scalers( + static_df, scalers["static"], static_to_scale, static_scaler_type + ) + else: + static_features = static_df + + # Step 9: Adjust feature names for shifted moving averages + adjusted_features = adjust_feature_names(features, moving_windows, shift_moving_avg) + + # Step 10: Build separate TimeSeries per basin + target_list: list[TimeSeries] = [] + past_cov_list: list[TimeSeries] = [] + future_cov_list: list[TimeSeries] = [] + codes_list: list[int] = [] + + # Resolve available past/future feature columns + available_past = [f for f in adjusted_features if f in forcing_df.columns] + available_future = [f for f in future_features if f in forcing_df.columns] + + if not available_past and past_features: + logger.warning(f"No past feature columns found. Requested: {adjusted_features}") + if not available_future and future_features: + logger.warning(f"No future feature columns found. Requested: {future_features}") + + for code in discharge_df[basin_id_col].unique(): + code = int(code) + dis_basin = discharge_df[discharge_df[basin_id_col] == code].copy() + frc_basin = forcing_df[forcing_df[basin_id_col] == code].copy() + + # Determine t = last non-NaN discharge date + valid_discharge = dis_basin.dropna(subset=[target_col]) + if valid_discharge.empty: + logger.warning(f"Basin {code}: no valid discharge observations, skipping") + continue + + t = valid_discharge[date_col].max() + + # Filter target data: up to t, keep NaN gaps (model handles them) + dis_up_to_t = dis_basin[dis_basin[date_col] <= t].copy() + + # Build target TimeSeries + dis_up_to_t = dis_up_to_t.sort_values(date_col) + target_ts = TimeSeries.from_dataframe( + dis_up_to_t, time_col=date_col, value_cols=[target_col], freq="1D" + ).astype(np.float32) + + # Add static covariates to target + if code in static_features.index: + target_ts = target_ts.with_static_covariates(static_features.loc[code]) + + target_list.append(target_ts) + codes_list.append(code) + + # Build past_covariates: up to t + if available_past: + frc_up_to_t = frc_basin[frc_basin[date_col] <= t].sort_values(date_col) + past_cov_ts = TimeSeries.from_dataframe( + frc_up_to_t, time_col=date_col, value_cols=available_past, freq="1D" + ).astype(np.float32) + past_cov_list.append(past_cov_ts) + + # Build future_covariates: full range + if available_future: + frc_full = frc_basin.sort_values(date_col) + future_cov_ts = TimeSeries.from_dataframe( + frc_full, time_col=date_col, value_cols=available_future, freq="1D" + ).astype(np.float32) + future_cov_list.append(future_cov_ts) + + logger.info( + f"Inference data: {len(target_list)} basins, " + f"past_cov={'yes' if past_cov_list else 'no'}, " + f"future_cov={'yes' if future_cov_list else 'no'}" + ) + + return { + "target": target_list, + "past_covariates": past_cov_list or None, + "future_covariates": future_cov_list or None, + "codes": codes_list, + } def _build_timeseries_pipeline( @@ -277,7 +497,7 @@ def _build_timeseries_pipeline( # Step 8: Scale static features if scale_static: static_features_config = config.get("static_features", []) - static_to_scale = _resolve_static_features( + static_to_scale = resolve_static_features( static_df, static_features_config, static_id_col ) if mode == "fit": @@ -286,16 +506,14 @@ def _build_timeseries_pipeline( ) computed_scalers["static"] = static_scalers else: - static_features = _apply_static_scalers( + static_features = apply_static_scalers( static_df, scalers["static"], static_to_scale, static_scaler_type ) else: static_features = static_df # Adjust feature names for shifted moving averages - adjusted_features = _adjust_feature_names( - features, moving_windows, shift_moving_avg - ) + adjusted_features = adjust_feature_names(features, moving_windows, shift_moving_avg) logger.info(f"Adjusted features: {adjusted_features}") # Step 9: Build Darts TimeSeries @@ -339,7 +557,7 @@ def _split_temporal_data( return discharge_df, forcing_df -def _apply_static_scalers( +def apply_static_scalers( static_df: pd.DataFrame, scalers: dict, features_to_scale: list[str], @@ -367,7 +585,7 @@ def _apply_static_scalers( return static_features -def _adjust_feature_names( +def adjust_feature_names( features: list[str], moving_windows: list[int], shift_moving_avg: bool, diff --git a/st_forecast/data_utils/processors.py b/st_forecast/data_utils/processors.py index b996f29..7ddc433 100644 --- a/st_forecast/data_utils/processors.py +++ b/st_forecast/data_utils/processors.py @@ -9,7 +9,7 @@ import numpy as np import logging -from st_forecast.data_utils.transformers import reindex_dataframe, process_sla_data +from st_forecast.data_utils.transformers import reindex_dataframe logger = logging.getLogger(__name__) diff --git a/st_forecast/data_utils/scalers.py b/st_forecast/data_utils/scalers.py index 1aaafd7..6b78df8 100644 --- a/st_forecast/data_utils/scalers.py +++ b/st_forecast/data_utils/scalers.py @@ -5,7 +5,6 @@ import numpy as np import pandas as pd -from sklearn.preprocessing import MinMaxScaler, StandardScaler from ..log_config import setup_logging diff --git a/st_forecast/evaluate_model.py b/st_forecast/evaluate_model.py index 15c601d..18e178b 100644 --- a/st_forecast/evaluate_model.py +++ b/st_forecast/evaluate_model.py @@ -1,23 +1,9 @@ -import os -import sys -import json -import pandas as pd -import numpy as np -import matplotlib.pyplot as plt import pandas as pd import numpy as np from darts import TimeSeries, concatenate -import darts -from darts.models import TFTModel, TiDEModel, TSMixerModel import torch -import lightning.pytorch as pl -from pytorch_lightning.callbacks import Callback -from pytorch_lightning.callbacks import EarlyStopping -import torch.nn as nn -from torch.optim.lr_scheduler import StepLR -import random from darts.metrics import rmse, r2_score @@ -117,9 +103,7 @@ def create_prediction_df( data = { "date": predictions.time_index, **{ - f"Q{int(q * 100)}": predictions.quantile(q) - .values() - .flatten() + f"Q{int(q * 100)}": predictions.quantile(q).values().flatten() for q in quantiles }, } diff --git a/st_forecast/load_data_darts.py b/st_forecast/load_data_darts.py index 279e0b6..0632465 100644 --- a/st_forecast/load_data_darts.py +++ b/st_forecast/load_data_darts.py @@ -1,11 +1,7 @@ import pandas as pd import numpy as np -import plotly.express as px -import darts -from darts import TimeSeries, concatenate -from darts.utils.timeseries_generation import datetime_attribute_timeseries +from darts import TimeSeries from st_forecast.utils import preprocessing_darts -from scipy.signal import savgol_filter def load_data( @@ -381,7 +377,6 @@ def stack_era5(discharge, forcing, exogene_features): # ----------------- # from darts.utils.missing_values import extract_subseries, missing_values_ratio - from darts.dataprocessing.transformers import MissingValuesFiller time_series_without_missing = [ ts for ts in discharge_train if missing_values_ratio(ts["discharge"]) == 0 diff --git a/st_forecast/operational/base_predictor.py b/st_forecast/operational/base_predictor.py index 2fc6e96..cef1c8f 100644 --- a/st_forecast/operational/base_predictor.py +++ b/st_forecast/operational/base_predictor.py @@ -4,13 +4,8 @@ """ from abc import ABC, abstractmethod -import os import pandas as pd -import numpy as np -import matplotlib.pyplot as plt -import darts -from darts import TimeSeries -from typing import List, Dict, Tuple, Union +from typing import Union class BasePredictor(ABC): diff --git a/st_forecast/operational/sapphire_predictor.py b/st_forecast/operational/sapphire_predictor.py index 618b251..1bd5f47 100644 --- a/st_forecast/operational/sapphire_predictor.py +++ b/st_forecast/operational/sapphire_predictor.py @@ -12,11 +12,14 @@ from darts import TimeSeries from pytorch_lightning import Trainer +from st_forecast.data_utils.feature_engineers import add_derived_features from st_forecast.data_utils.preparation import ( - extract_timeseries_components, - prepare_inference_data, + adjust_feature_names, + apply_static_scalers, + resolve_static_features, ) -from st_forecast.data_utils.scalers import apply_era5_scalers +from st_forecast.data_utils.processors import transform_discharge_units +from st_forecast.data_utils.scalers import apply_discharge_scalers, apply_era5_scalers from st_forecast.operational.base_predictor import BasePredictor from st_forecast.pipeline.artifacts import load_run_artifacts from st_forecast.pipeline.inference import ( @@ -33,13 +36,15 @@ logging.basicConfig(level=logging.WARNING) warnings.filterwarnings("ignore") +logger = logging.getLogger(__name__) + class SapphirePredictor(BasePredictor): - """ - Darts-based Deep Learning predictor for SAPPHIRE discharge forecasting. + """Darts-based Deep Learning predictor for SAPPHIRE discharge forecasting. - Wraps ST-Forecast internal functions for model loading, data preparation, - and inference on a single-basin basis. + Follows the BaseDartsDLPredictor pattern: keeps discharge and covariates + as separate DataFrames, builds 3 separate TimeSeries (target, past_covariates, + future_covariates) with appropriate date ranges. """ def __init__(self, model_path: str): @@ -56,7 +61,7 @@ def __init__(self, model_path: str): self.static_df = pd.read_csv( static_path, index_col=self.config.static_id_col ) - # Ensure static_id_col exists as a column (needed for extract_timeseries_components) + # Ensure static_id_col exists as a column (needed for downstream use) static_id = self.config.static_id_col if static_id not in self.static_df.columns: self.static_df[static_id] = self.static_df.index @@ -74,12 +79,180 @@ def __init__(self, model_path: str): self.quantiles = self.config.quantiles or DEFAULT_QUANTILES self.num_samples = self.config.num_samples + # Pre-scale static features once at init + if self.static_df is not None and self.scalers.static: + static_features_list = resolve_static_features( + self.static_df, + self.config.static_features, + self.config.static_id_col, + ) + self.scaled_static_df = apply_static_scalers( + self.static_df, + self.scalers.static, + static_features_list, + self.config.static_scaler_type, + ) + # Remove the CODE column so it doesn't pollute TimeSeries static covariates + if self.config.static_id_col in self.scaled_static_df.columns: + self.scaled_static_df = self.scaled_static_df.drop( + columns=[self.config.static_id_col] + ) + else: + self.scaled_static_df = None + def get_input_chunk_length(self) -> int: return self.config.input_chunk_length def get_max_forecast_horizon(self) -> int: return self.config.forecast_horizon + def _prepare_data( + self, + past_measurements: pd.DataFrame, + covariates: pd.DataFrame, + code: int, + ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + """Prepare and scale data for model input. + + Follows the BaseDartsDLPredictor pattern: keeps discharge and covariates + as separate DataFrames throughout. The calling scripts handle missing-value + imputation and date filtering before calling the predictor. + + Returns + ------- + tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame] + (df_discharge, df_covariates, df_covariates_past) + """ + date_col = self.config.date_col + basin_id_col = self.config.basin_id_col + target_col = self.config.target_col + + # Data arrives pre-filtered to a single basin by the calling script. + # Just copy, ensure datetime, and sort. + df_discharge = past_measurements.copy() + df_discharge[date_col] = pd.to_datetime(df_discharge[date_col]) + df_discharge = df_discharge.sort_values(date_col) + + df_covariates = covariates.copy() + df_covariates[date_col] = pd.to_datetime(df_covariates[date_col]) + df_covariates = df_covariates.sort_values(date_col) + + # Derive PET and daylight_hours if missing + if ( + "PET" not in df_covariates.columns + or "daylight_hours" not in df_covariates.columns + ): + if self.static_df is not None: + df_covariates = add_derived_features( + df_covariates, + self.static_df, + date_col=date_col, + basin_id_col=basin_id_col, + ) + + # Unit-transform discharge (mm conversion, log transform) + if self.config.transform_mm or self.config.transform_log: + df_discharge = transform_discharge_units( + df_discharge, + self.static_df, + self.config.transform_mm, + self.config.transform_log, + ) + + # Scale discharge + if self.config.scale_discharge_bool and self.scalers.discharge: + df_discharge = apply_discharge_scalers( + df_discharge, + self.scalers.discharge, + self.config.discharge_scaler_type, + self.config.global_scaling, + ) + + # Scale covariates (exog features) + if self.config.exog_features and self.scalers.era5: + df_covariates = apply_era5_scalers( + df_covariates, + self.scalers.era5, + self.config.exog_features, + self.config.forcing_scaler_type, + ) + + # Compute past covariates (rolling means on scaled discharge) + df_covariates_past = pd.DataFrame({date_col: df_discharge[date_col].values}) + for window in self.config.moving_windows: + rolling = df_discharge[target_col].rolling(window=window).mean() + if self.config.shift_moving_avg: + rolling = rolling.shift(periods=self.config.forecast_horizon) + rolling = rolling.bfill() + suffix = "_shifted" if self.config.shift_moving_avg else "" + col = f"moving_avr_dis_{window}{suffix}" + df_covariates_past[col] = rolling.values + + return df_discharge, df_covariates, df_covariates_past + + def _create_time_series( + self, + df_discharge: pd.DataFrame, + df_covariates: pd.DataFrame, + df_covariates_past: pd.DataFrame, + code: int, + ) -> tuple[TimeSeries, TimeSeries | None, TimeSeries | None]: + """Create Darts TimeSeries objects from prepared DataFrames. + + Returns + ------- + tuple[TimeSeries, TimeSeries | None, TimeSeries | None] + (target_ts, past_cov_ts, future_cov_ts) + """ + date_col = self.config.date_col + target_col = self.config.target_col + + # Target TimeSeries (discharge) + target_ts = TimeSeries.from_dataframe( + df_discharge, time_col=date_col, value_cols=[target_col], freq="1D" + ).astype(np.float32) + + # Attach scaled static covariates + if self.scaled_static_df is not None and code in self.scaled_static_df.index: + target_ts = target_ts.with_static_covariates( + self.scaled_static_df.loc[code] + ) + + # Past covariates TimeSeries (rolling means) + past_feature_cols = adjust_feature_names( + self.config.past_features, + self.config.moving_windows, + self.config.shift_moving_avg, + ) + available_past = [ + c for c in past_feature_cols if c in df_covariates_past.columns + ] + if available_past: + past_cov_ts = TimeSeries.from_dataframe( + df_covariates_past, + time_col=date_col, + value_cols=available_past, + freq="1D", + ).astype(np.float32) + else: + past_cov_ts = None + + # Future covariates TimeSeries (weather features, full date range) + available_future = [ + c for c in self.config.future_features if c in df_covariates.columns + ] + if available_future: + future_cov_ts = TimeSeries.from_dataframe( + df_covariates, + time_col=date_col, + value_cols=available_future, + freq="1D", + ).astype(np.float32) + else: + future_cov_ts = None + + return target_ts, past_cov_ts, future_cov_ts + def predict( self, past_measurements: pd.DataFrame, @@ -92,7 +265,7 @@ def predict( Parameters ---------- past_measurements : pd.DataFrame - Past discharge observations for a single basin + Past discharge observations for a single basin (clean, no NaN gaps) covariates : pd.DataFrame Covariate data (weather, etc.) extending into future identifier : int | str @@ -107,74 +280,25 @@ def predict( """ if n > self.config.forecast_horizon: logging.warning( - f"Requested horizon n={n} exceeds output_chunk_length={self.config.forecast_horizon}. " - f"Capping to {self.config.forecast_horizon} - this can lead to auto-regression issues and degraded performance." + f"Requested horizon n={n} exceeds output_chunk_length=" + f"{self.config.forecast_horizon}. Capping to " + f"{self.config.forecast_horizon}." ) n = self.config.forecast_horizon code = int(identifier) - date_col = self.config.date_col - basin_id_col = self.config.basin_id_col - # Inner merge for historical data (where we have both discharge and covariates) - temporal_df = pd.merge( - past_measurements, - covariates, - on=[date_col, basin_id_col], - how="inner", + # Prepare DataFrames + df_discharge, df_covariates, df_covariates_past = self._prepare_data( + past_measurements, covariates, code ) - # Prepare scalers dict - scalers_dict = { - "discharge": self.scalers.discharge, - "era5": self.scalers.era5, - "static": self.scalers.static, - } - - # Prepare config with column names - prepare_config = self.config.to_prepare_config() - prepare_config["date_col"] = date_col - prepare_config["basin_id_col"] = basin_id_col - prepare_config["target_col"] = self.config.target_col - prepare_config["static_id_col"] = self.config.static_id_col - - # Prepare data using scalers (this handles target + past_covariates) - data_result = prepare_inference_data( - temporal_df, self.static_df, scalers_dict, prepare_config + # Create TimeSeries + target_ts, past_cov_ts, future_cov_ts = self._create_time_series( + df_discharge, df_covariates, df_covariates_past, code ) - timeseries = data_result["timeseries"] - if not timeseries: - raise ValueError(f"No valid timeseries for basin {code}") - - # Extract timeseries components - target, past_cov, future_cov, _ = extract_timeseries_components( - timeseries, - self.config.past_features, - self.config.future_features, - target_col=self.config.target_col, - static_basin_id_col=self.config.static_id_col, - ) - - # Use last segment (most recent) for forecasting - idx = -1 - target_ts = target[idx] - past_cov_ts = past_cov[idx] if past_cov else None - - # For operational forecasting, build extended future_covariates - # that includes data beyond the forecast date - future_cov_ts = None - if self.config.future_features and future_cov: - future_cov_ts = self._build_extended_future_covariates( - covariates=covariates, - target_ts=target_ts, - code=code, - n=n, - ) - - # Create trainer for CPU inference - trainer = Trainer(accelerator="cpu", logger=False) - # Run model.predict() + trainer = Trainer(accelerator="cpu", logger=False) prediction = self.model.predict( n=n, series=target_ts, @@ -185,87 +309,15 @@ def predict( mc_dropout=self.config.mc_dropout, ) - # Convert to DataFrame and inverse scale - pred_df = create_prediction_df(prediction, self.quantiles, date_col=date_col) - pred_df[basin_id_col] = code - pred_df = inverse_scale_predictions( + # Post-process: quantiles + inverse scaling + pred_df = create_prediction_df( + prediction, self.quantiles, date_col=self.config.date_col + ) + pred_df[self.config.basin_id_col] = code + return inverse_scale_predictions( pred_df, self.scalers, self.config, self.static_df ) - return pred_df - - def _build_extended_future_covariates( - self, - covariates: pd.DataFrame, - target_ts: TimeSeries, - code: int, - n: int, - ) -> TimeSeries: - """Build future covariates TimeSeries that extends beyond the target series. - - Parameters - ---------- - covariates : pd.DataFrame - Full covariate data including future dates - target_ts : TimeSeries - Target series (to determine required time range) - code : int - Basin code - n : int - Forecast horizon - - Returns - ------- - TimeSeries - Future covariates extending from (target_start - input_chunk) to (target_end + n) - """ - date_col = self.config.date_col - basin_id_col = self.config.basin_id_col - future_features = self.config.future_features - - # Determine required time range - target_end = target_ts.end_time() - target_start = target_ts.start_time() - required_end = target_end + pd.Timedelta(days=n) - - # Filter covariates for this basin and required time range - basin_cov = covariates[covariates[basin_id_col] == code].copy() - basin_cov = basin_cov[ - (basin_cov[date_col] >= target_start) - & (basin_cov[date_col] <= required_end) - ].copy() - - if basin_cov.empty: - raise ValueError(f"No covariate data for basin {code} in required range") - - # Check we have enough future data - max_cov_date = basin_cov[date_col].max() - if max_cov_date < required_end: - raise ValueError( - f"Covariates only extend to {max_cov_date}, but need data until {required_end} " - f"for {n}-day forecast. Please provide covariates extending {n} days into the future." - ) - - # Scale the future features using ERA5 scalers - if self.scalers.era5: - basin_cov = apply_era5_scalers( - basin_cov, - self.scalers.era5, - future_features, - self.scalers.era5_scaler_type, - ) - - # Create TimeSeries for future covariates - basin_cov = basin_cov.sort_values(date_col) - future_cov_ts = TimeSeries.from_dataframe( - basin_cov, - time_col=date_col, - value_cols=future_features, - freq="1D", - ).astype(np.float32) - - return future_cov_ts - def hindcast( self, past_measurements: pd.DataFrame, @@ -280,8 +332,8 @@ def hindcast( past_measurements : pd.DataFrame Past discharge observations for a single basin covariates : pd.DataFrame - Covariate data (weather, etc.) - should extend n days beyond past_measurements - to enable hindcasts at the end of the period + Covariate data (weather, etc.) — should extend n days beyond + past_measurements to enable hindcasts at the end of the period identifier : int | str Basin code identifier n : int @@ -291,101 +343,55 @@ def hindcast( ------- pd.DataFrame Hindcast predictions with columns: - date, code, forecast_step, forecast_issue_date, Q5, Q10, Q25, Q50, Q75, Q90, Q95 + date, code, forecast_step, forecast_issue_date, Q5..Q95 """ if n > self.config.forecast_horizon: logging.warning( - f"Requested horizon n={n} exceeds output_chunk_length={self.config.forecast_horizon}. " - f"Capping to {self.config.forecast_horizon} - this can lead to auto-regression issues and degraded performance." + f"Requested horizon n={n} exceeds output_chunk_length=" + f"{self.config.forecast_horizon}. Capping to " + f"{self.config.forecast_horizon}." ) n = self.config.forecast_horizon code = int(identifier) date_col = self.config.date_col basin_id_col = self.config.basin_id_col - # Inner merge for historical data (where we have both discharge and covariates) - temporal_df = pd.merge( - past_measurements, - covariates, - on=[date_col, basin_id_col], - how="inner", - ) - - # Prepare scalers dict - scalers_dict = { - "discharge": self.scalers.discharge, - "era5": self.scalers.era5, - "static": self.scalers.static, - } - - # Prepare config with column names - prepare_config = self.config.to_prepare_config() - prepare_config["date_col"] = date_col - prepare_config["basin_id_col"] = basin_id_col - prepare_config["target_col"] = self.config.target_col - prepare_config["static_id_col"] = self.config.static_id_col - - # Prepare data using scalers - data_result = prepare_inference_data( - temporal_df, self.static_df, scalers_dict, prepare_config + # Prepare DataFrames (same as predict — no mode distinction) + df_discharge, df_covariates, df_covariates_past = self._prepare_data( + past_measurements, covariates, code ) - timeseries = data_result["timeseries"] - if not timeseries: - raise ValueError(f"No valid timeseries for basin {code}") - - # Extract timeseries components - target, past_cov, future_cov, _ = extract_timeseries_components( - timeseries, - self.config.past_features, - self.config.future_features, - target_col=self.config.target_col, - static_basin_id_col=self.config.static_id_col, + # Create TimeSeries + target_ts, past_cov_ts, future_cov_ts = self._create_time_series( + df_discharge, df_covariates, df_covariates_past, code ) - # Create trainer for CPU inference + # Run model.historical_forecasts() predict_kwargs = { "trainer": Trainer(accelerator="cpu", logger=False), "mc_dropout": self.config.mc_dropout, } + backtests = self.model.historical_forecasts( + series=target_ts, + past_covariates=past_cov_ts, + future_covariates=future_cov_ts, + forecast_horizon=n, + num_samples=self.num_samples, + retrain=False, + stride=1, + last_points_only=False, + verbose=False, + predict_kwargs=predict_kwargs, + ) - # Run hindcast on all segments (handles gaps in data) - all_hindcast_dfs = [] - for i in range(len(target)): - target_ts = target[i] - past_cov_ts = past_cov[i] if past_cov else None - - # Build extended future covariates that include data beyond the target series - # This ensures hindcasts near the end of the series have enough future covariate data - future_cov_ts = None - if self.config.future_features and future_cov: - future_cov_ts = self._build_extended_future_covariates( - covariates=covariates, - target_ts=target_ts, - code=code, - n=n, - ) - - backtests = self.model.historical_forecasts( - series=target_ts, - past_covariates=past_cov_ts, - future_covariates=future_cov_ts, - forecast_horizon=n, - num_samples=self.num_samples, - retrain=False, - stride=1, - last_points_only=False, - verbose=False, - predict_kwargs=predict_kwargs, - ) - - # Process results: add forecast_step - for ts in backtests: - df = create_prediction_df(ts, self.quantiles, date_col=date_col) - df["forecast_step"] = range(1, len(df) + 1) - all_hindcast_dfs.append(df) + # Process results + all_dfs = [] + for ts in backtests: + df = create_prediction_df(ts, self.quantiles, date_col=date_col) + df["forecast_step"] = range(1, len(df) + 1) + all_dfs.append(df) - result = pd.concat(all_hindcast_dfs, ignore_index=True) + result = pd.concat(all_dfs, ignore_index=True) result[basin_id_col] = code result["forecast_issue_date"] = result[date_col] - pd.to_timedelta( result["forecast_step"], unit="D" diff --git a/st_forecast/optimize.py b/st_forecast/optimize.py index e4560ea..3dd9678 100644 --- a/st_forecast/optimize.py +++ b/st_forecast/optimize.py @@ -1,27 +1,14 @@ import os -import sys import json import pandas as pd import numpy as np -import matplotlib.pyplot as plt -import darts -from darts import TimeSeries, concatenate -from darts.utils.timeseries_generation import datetime_attribute_timeseries -from darts.models import TFTModel, TiDEModel, TSMixerModel import torch -import lightning.pytorch as pl -from pytorch_lightning.callbacks import Callback -from pytorch_lightning.callbacks import EarlyStopping - -import torch.nn as nn -from torch.optim.lr_scheduler import StepLR import warnings import logging import random import optuna -from optuna.integration import PyTorchLightningPruningCallback from darts.utils.likelihood_models import QuantileRegression @@ -33,7 +20,6 @@ from st_forecast import get_data from st_forecast import model_helper from st_forecast import evaluate_model -from st_forecast import visualization def prepare_data_run(config_data, this_try, config_optuna): @@ -232,9 +218,6 @@ def objective_TSMixer(trial): pass -from optuna_dashboard import run_server - - def run_optuna(config_data, config_optuna): # Set the random seed for reproducibility random.seed(42) diff --git a/st_forecast/pipeline/artifacts.py b/st_forecast/pipeline/artifacts.py index 3ae6095..7545821 100644 --- a/st_forecast/pipeline/artifacts.py +++ b/st_forecast/pipeline/artifacts.py @@ -223,6 +223,8 @@ def to_prepare_config(self) -> dict: "codes_to_remove": self.codes_to_remove, "unnatural_rivers": self.unnatural_rivers, "use_only_natural_rivers": self.use_only_natural_rivers, + "past_features": self.past_features, + "future_features": self.future_features, } def to_model_config_dict(self) -> dict: diff --git a/st_forecast/pipeline/forecast.py b/st_forecast/pipeline/forecast.py index bf1dfd4..b03d6a2 100644 --- a/st_forecast/pipeline/forecast.py +++ b/st_forecast/pipeline/forecast.py @@ -8,7 +8,7 @@ from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, ) from st_forecast.pipeline.artifacts import load_run_artifacts, RunConfig, Scalers from st_forecast.pipeline.inference import ( @@ -170,7 +170,7 @@ def _generate_forecast( } # Prepare data using inference mode (apply pre-computed scalers) - data_result = prepare_inference_data( + data_result = prepare_validation_data( temporal_df, static_df, scalers_dict, prepare_config ) timeseries = data_result["timeseries"] diff --git a/st_forecast/pipeline/hindcast.py b/st_forecast/pipeline/hindcast.py index cafc4c1..9c7994b 100644 --- a/st_forecast/pipeline/hindcast.py +++ b/st_forecast/pipeline/hindcast.py @@ -7,7 +7,7 @@ from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, ) from st_forecast.pipeline.artifacts import load_run_artifacts from st_forecast.pipeline.inference import ( @@ -104,7 +104,7 @@ def hindcast( "static": scalers.static, } - data_result = prepare_inference_data( + data_result = prepare_validation_data( temporal_df, static_df, scalers_dict, prepare_config ) timeseries = data_result["timeseries"] diff --git a/st_forecast/pipeline/inference.py b/st_forecast/pipeline/inference.py index 179c07c..d0870d6 100644 --- a/st_forecast/pipeline/inference.py +++ b/st_forecast/pipeline/inference.py @@ -116,9 +116,7 @@ def create_prediction_df( data = { date_col: predictions.time_index, **{ - f"Q{int(q * 100)}": predictions.quantile(q) - .values() - .flatten() + f"Q{int(q * 100)}": predictions.quantile(q).values().flatten() for q in quantiles }, } diff --git a/st_forecast/pipeline/workflow.py b/st_forecast/pipeline/workflow.py index 7c3a9a9..f6dcdc4 100644 --- a/st_forecast/pipeline/workflow.py +++ b/st_forecast/pipeline/workflow.py @@ -8,7 +8,7 @@ from st_forecast.data_utils.preparation import ( extract_timeseries_components, - prepare_inference_data, + prepare_validation_data, prepare_train_data, ) from st_forecast.pipeline.artifacts import RunConfig, Scalers @@ -144,7 +144,9 @@ def train_model( scalers_dict = train_result["scalers"] # Prepare validation data (applies existing scalers) - val_result = prepare_inference_data(val_df, static_df, scalers_dict, prepare_config) + val_result = prepare_validation_data( + val_df, static_df, scalers_dict, prepare_config + ) # Extract TimeSeries components target_train, past_cov_train, future_cov_train, _ = extract_timeseries_components( diff --git a/st_forecast/train_model.py b/st_forecast/train_model.py index 1c6a92a..36a4821 100644 --- a/st_forecast/train_model.py +++ b/st_forecast/train_model.py @@ -51,7 +51,7 @@ from st_forecast.data_utils.loaders import load_data_df, RawDataCollector from st_forecast.data_utils.preparation import ( prepare_train_data, - prepare_inference_data, + prepare_validation_data, ) from st_forecast.custom_models.FANMixer import FANMixerModel from st_forecast.custom_models.ExoMamba import ExoMambaModel @@ -59,7 +59,7 @@ def _build_prepare_config(data_config: dict, model_config: dict) -> dict: - """Build config dict for prepare_train_data/prepare_inference_data. + """Build config dict for prepare_train_data/prepare_validation_data. Merges relevant settings from both data and model configs into the format expected by the preparation module. @@ -87,7 +87,7 @@ def _build_prepare_config(data_config: dict, model_config: dict) -> dict: def _load_data_new_pipeline(data_config: dict, model_config: dict) -> dict: - """Load data using the new prepare_train_data/prepare_inference_data functions. + """Load data using the new prepare_train_data/prepare_validation_data functions. This implements the new modular data preparation pipeline that separates training (fit scalers) from inference (apply scalers). @@ -172,7 +172,7 @@ def _load_data_new_pipeline(data_config: dict, model_config: dict) -> dict: # Step 6: Prepare validation data (applies scalers) if val_temporal is not None: - val_result = prepare_inference_data( + val_result = prepare_validation_data( val_temporal, static_df, scalers, prepare_config ) val_timeseries = val_result["timeseries"] @@ -180,7 +180,7 @@ def _load_data_new_pipeline(data_config: dict, model_config: dict) -> dict: val_timeseries = None # Step 7: Prepare test data (applies scalers) - test_result = prepare_inference_data( + test_result = prepare_validation_data( test_temporal, static_df, scalers, prepare_config ) test_timeseries = test_result["timeseries"] @@ -244,7 +244,7 @@ def train_model(config_data_path, config_model_path): use_new_prepare_pipeline = config_model.get("use_new_prepare_pipeline", False) if use_new_prepare_pipeline: - # Use the new prepare_train_data/prepare_inference_data functions + # Use the new prepare_train_data/prepare_validation_data functions # This provides clean separation between training (fit scalers) and inference (apply scalers) logging.info("Using new modular prepare pipeline") data_dict = _load_data_new_pipeline( diff --git a/st_forecast/transfer_fine_tuning.py b/st_forecast/transfer_fine_tuning.py index fbf6196..58d8ae9 100644 --- a/st_forecast/transfer_fine_tuning.py +++ b/st_forecast/transfer_fine_tuning.py @@ -1,23 +1,11 @@ import os -import sys import json import pandas as pd import numpy as np -import matplotlib.pyplot as plt -from darts import TimeSeries, concatenate -from darts.utils.timeseries_generation import datetime_attribute_timeseries -import darts -from darts.models import TFTModel, TiDEModel, TSMixerModel +from darts.models import TiDEModel import torch -import lightning.pytorch as pl -from pytorch_lightning.callbacks import Callback -from pytorch_lightning.callbacks import EarlyStopping - -import torch.nn as nn -from torch.optim.lr_scheduler import StepLR -import warnings -import logging + import random from darts.utils.likelihood_models import QuantileRegression @@ -29,7 +17,6 @@ from st_forecast import model_helper from st_forecast import evaluate_model from st_forecast import visualization -from st_forecast.model_helper import LossLogger def load_tide_model(model_folder, model_path, config_model, delta_input_dims): diff --git a/st_forecast/utils/preprocessing_darts.py b/st_forecast/utils/preprocessing_darts.py index ac50b37..6c43fa2 100644 --- a/st_forecast/utils/preprocessing_darts.py +++ b/st_forecast/utils/preprocessing_darts.py @@ -9,9 +9,6 @@ import pandas as pd import numpy as np -import darts -from darts import TimeSeries, concatenate -from darts.utils.timeseries_generation import datetime_attribute_timeseries from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import StandardScaler diff --git a/st_forecast/visualization/_helpers.py b/st_forecast/visualization/_helpers.py index 08e0229..8adf942 100644 --- a/st_forecast/visualization/_helpers.py +++ b/st_forecast/visualization/_helpers.py @@ -121,7 +121,7 @@ def load_multiple_run_metrics( for key, df in run_data.items(): if not df.empty: df = df.copy() - df["run"] = label + df["run"] = label all_metrics[key].append(df) result = {} diff --git a/st_forecast/visualization/_styles.py b/st_forecast/visualization/_styles.py index f34db3e..6f07f75 100644 --- a/st_forecast/visualization/_styles.py +++ b/st_forecast/visualization/_styles.py @@ -100,7 +100,7 @@ # Explicit model color overrides for visual distinction MODEL_COLORS: dict[str, str] = { - "lstm": "#4477AA", # blue + "lstm": "#4477AA", # blue "ensemble": "#AA3377", # purple } diff --git a/st_forecast/visualization/compare_runs.py b/st_forecast/visualization/compare_runs.py index 45f0a68..6815fa4 100644 --- a/st_forecast/visualization/compare_runs.py +++ b/st_forecast/visualization/compare_runs.py @@ -228,11 +228,11 @@ def plot_calibration_comparison( fig, ax = plt.subplots(figsize=(8, 8)) - ax.plot( - [0, 1], [0, 1], "k--", linewidth=1.5, label="Perfect Calibration", alpha=0.7 - ) + ax.plot([0, 1], [0, 1], "k-", linewidth=2, label="Perfect Calibration", zorder=1) + + markers = ["o", "s", "D", "^", "v", "P", "X", "*", "h", "<", ">"] - for run_label in labels_no_persistence: + for i, run_label in enumerate(labels_no_persistence): run_data = prob_df[prob_df["run"] == run_label] if run_data.empty: continue @@ -251,12 +251,13 @@ def plot_calibration_comparison( ax.plot( theoretical_rates, empirical_rates, - marker="o", + marker=markers[i % len(markers)], color=palette[run_label], linewidth=0.8, linestyle="--", markersize=8, label=run_label, + zorder=2, ) ax.set_xlabel("Theoretical Exceedance Rate") @@ -344,7 +345,7 @@ def plot_crps_comparison( ) ax.set_xlabel("Forecast Step [days]") - ax.set_ylabel("CRPS [-]") + ax.set_ylabel("nCRPS [-]") ax.set_title("") ax.legend(title=None, bbox_to_anchor=(1.02, 1), loc="upper left") diff --git a/tests/test_evaluation/test_probabilistic_metrics.py b/tests/test_evaluation/test_probabilistic_metrics.py index aa2d98b..3796b72 100644 --- a/tests/test_evaluation/test_probabilistic_metrics.py +++ b/tests/test_evaluation/test_probabilistic_metrics.py @@ -10,8 +10,6 @@ compute_basin_crps, compute_all_basins_probabilistic, save_probabilistic_metrics, - DEFAULT_QUANTILE_COLS, - DEFAULT_QUANTILE_LEVELS, ) diff --git a/tests/test_imports.py b/tests/test_imports.py index e2ac504..5b3da4e 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -1,7 +1,5 @@ """Smoke tests to verify refactored imports work correctly.""" -import pytest - def test_data_utils_imports(): """Test that data_utils module imports successfully.""" diff --git a/tests/test_inference.py b/tests/test_inference.py index e0d21e0..5481c7e 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -2,7 +2,7 @@ import tempfile from pathlib import Path -from unittest.mock import Mock, MagicMock, patch +from unittest.mock import Mock, patch import numpy as np import pandas as pd diff --git a/tests/test_preparation.py b/tests/test_preparation.py index c691833..ea04fdb 100644 --- a/tests/test_preparation.py +++ b/tests/test_preparation.py @@ -8,9 +8,9 @@ prepare_train_data, prepare_inference_data, _split_temporal_data, - _adjust_feature_names, - _apply_static_scalers, - _resolve_static_features, + adjust_feature_names, + apply_static_scalers, + resolve_static_features, ) @@ -54,13 +54,13 @@ def test_preserves_month_column_in_discharge(self): class TestAdjustFeatureNames: def test_no_change_when_shift_disabled(self): features = ["T", "P", "moving_avr_dis_7", "moving_avr_dis_14"] - result = _adjust_feature_names(features, [7, 14], shift_moving_avg=False) + result = adjust_feature_names(features, [7, 14], shift_moving_avg=False) assert result == features def test_adds_shifted_suffix_when_enabled(self): features = ["T", "P", "moving_avr_dis_7", "moving_avr_dis_14"] - result = _adjust_feature_names(features, [7, 14], shift_moving_avg=True) + result = adjust_feature_names(features, [7, 14], shift_moving_avg=True) assert "moving_avr_dis_7_shifted" in result assert "moving_avr_dis_14_shifted" in result @@ -84,7 +84,7 @@ def test_minmax_scaling(self): "elevation": (1000.0, 3000.0), } - result = _apply_static_scalers( + result = apply_static_scalers( static_df, scalers, ["area", "elevation"], "minmax" ) @@ -106,7 +106,7 @@ def test_standard_scaling(self): std = 100.0 scalers = {"area": (mean, std)} - result = _apply_static_scalers(static_df, scalers, ["area"], "standard") + result = apply_static_scalers(static_df, scalers, ["area"], "standard") assert result["area"].iloc[0] == pytest.approx(-1.0) assert result["area"].iloc[1] == pytest.approx(0.0) @@ -123,12 +123,12 @@ def test_handles_zero_denominator(self): # minmax with min == max scalers = {"constant": (5.0, 5.0)} - result = _apply_static_scalers(static_df, scalers, ["constant"], "minmax") + result = apply_static_scalers(static_df, scalers, ["constant"], "minmax") assert result["constant"].iloc[0] == 0.0 # standard with std == 0 scalers = {"constant": (5.0, 0.0)} - result = _apply_static_scalers(static_df, scalers, ["constant"], "standard") + result = apply_static_scalers(static_df, scalers, ["constant"], "standard") assert result["constant"].iloc[0] == 0.0 @@ -265,10 +265,10 @@ def config(self): def test_requires_scalers(self, sample_data, config): temporal_df, static_df = sample_data - with pytest.raises(ValueError, match="scalers must be provided"): + with pytest.raises(TypeError): prepare_inference_data(temporal_df, static_df, None, config) - def test_returns_only_timeseries(self, sample_data, config): + def test_returns_separate_components(self, sample_data, config): temporal_df, static_df = sample_data # First get scalers from training @@ -278,7 +278,11 @@ def test_returns_only_timeseries(self, sample_data, config): # Then use them for inference result = prepare_inference_data(temporal_df, static_df, scalers, config) - assert "timeseries" in result + assert "target" in result + assert "past_covariates" in result + assert "future_covariates" in result + assert "codes" in result + assert len(result["target"]) > 0 assert "scalers" not in result @@ -297,7 +301,7 @@ def static_df(self): return df def test_empty_list_uses_all_columns(self, static_df): - result = _resolve_static_features(static_df, [], "CODE") + result = resolve_static_features(static_df, [], "CODE") assert "area" in result assert "elevation" in result @@ -305,7 +309,7 @@ def test_empty_list_uses_all_columns(self, static_df): assert "CODE" not in result def test_explicit_list_filters_columns(self, static_df): - result = _resolve_static_features(static_df, ["area", "elevation"], "CODE") + result = resolve_static_features(static_df, ["area", "elevation"], "CODE") assert result == ["area", "elevation"] assert "slope" not in result @@ -314,7 +318,7 @@ def test_missing_columns_logs_warning(self, static_df, caplog): import logging with caplog.at_level(logging.WARNING): - result = _resolve_static_features( + result = resolve_static_features( static_df, ["area", "nonexistent_col"], "CODE" ) @@ -385,7 +389,8 @@ def test_inference_respects_config(self): # Then use them for inference result = prepare_inference_data(temporal_df, static_df, scalers, config) - assert "timeseries" in result + assert "target" in result + assert len(result["target"]) > 0 class TestPreparationIsRegionAgnostic: @@ -434,3 +439,124 @@ def test_no_region_in_config_required(self): assert "timeseries" in result assert len(result["timeseries"]) > 0 + + +class TestPrepareInferenceWithFutureCovariates: + """Test that inference works when covariates extend beyond discharge.""" + + def test_covariates_extend_beyond_discharge(self): + """Covariates extend 15 days beyond last discharge observation.""" + rng = np.random.default_rng(42) + discharge_days = 100 + future_days = 15 + total_days = discharge_days + future_days + + discharge_dates = pd.date_range("2020-01-01", periods=discharge_days, freq="D") + covariate_dates = pd.date_range("2020-01-01", periods=total_days, freq="D") + + rows = [] + for date in covariate_dates: + row = { + "date": date, + "code": 1, + "T": rng.random() * 30, + "P": rng.random() * 10, + } + if date in discharge_dates: + row["discharge"] = rng.random() * 100 + else: + row["discharge"] = np.nan + rows.append(row) + + temporal_df = pd.DataFrame(rows) + static_df = pd.DataFrame({"CODE": [1], "area": [1000.0], "elevation": [1500.0]}) + static_df.index = static_df["CODE"] + + config = { + "features": ["T", "P"], + "exog_features": ["T", "P"], + "past_features": [], + "future_features": ["T", "P"], + "moving_windows": [], + "shift_moving_avg": False, + "forecast_horizon": 6, + "input_chunk_length": 30, + "scale_discharge": False, + "scale_forcing": False, + "scale_static": False, + "transform_mm": False, + "transform_log": False, + "codes_to_remove": [], + "unnatural_rivers": [], + "use_only_natural_rivers": False, + } + + result = prepare_inference_data( + temporal_df, static_df, {"discharge": {}, "era5": {}, "static": {}}, config + ) + + assert len(result["target"]) == 1 + target_ts = result["target"][0] + assert target_ts.n_timesteps == discharge_days + + assert result["future_covariates"] is not None + future_ts = result["future_covariates"][0] + assert future_ts.n_timesteps == total_days + + def test_past_covariates_match_target_extent(self): + """Past covariates should end at the same time as the target.""" + rng = np.random.default_rng(42) + discharge_days = 80 + future_days = 10 + + discharge_dates = pd.date_range("2020-01-01", periods=discharge_days, freq="D") + covariate_dates = pd.date_range( + "2020-01-01", periods=discharge_days + future_days, freq="D" + ) + + rows = [] + for date in covariate_dates: + row = { + "date": date, + "code": 1, + "T": rng.random() * 30, + "P": rng.random() * 10, + } + if date in discharge_dates: + row["discharge"] = rng.random() * 100 + else: + row["discharge"] = np.nan + rows.append(row) + + temporal_df = pd.DataFrame(rows) + static_df = pd.DataFrame({"CODE": [1], "area": [1000.0]}) + static_df.index = static_df["CODE"] + + config = { + "features": ["T", "P"], + "exog_features": ["T", "P"], + "past_features": [], + "future_features": ["T", "P"], + "moving_windows": [], + "shift_moving_avg": False, + "forecast_horizon": 6, + "input_chunk_length": 30, + "scale_discharge": False, + "scale_forcing": False, + "scale_static": False, + "transform_mm": False, + "transform_log": False, + "codes_to_remove": [], + "unnatural_rivers": [], + "use_only_natural_rivers": False, + } + + result = prepare_inference_data( + temporal_df, static_df, {"discharge": {}, "era5": {}, "static": {}}, config + ) + + target_ts = result["target"][0] + past_cov_ts = result["past_covariates"][0] + + assert target_ts.end_time() == past_cov_ts.end_time() + assert target_ts.n_timesteps == discharge_days diff --git a/tests/test_region_specific_utils.py b/tests/test_region_specific_utils.py index 2d5e833..6da55d1 100644 --- a/tests/test_region_specific_utils.py +++ b/tests/test_region_specific_utils.py @@ -1,8 +1,6 @@ """Tests for region-specific utilities module.""" -import numpy as np import pandas as pd -import pytest from st_forecast.region_specific_utils import ( DISCHARGE_HANDLERS, diff --git a/uv.lock b/uv.lock index cb2756d..0003ad6 100644 --- a/uv.lock +++ b/uv.lock @@ -1,11 +1,20 @@ version = 1 revision = 2 -requires-python = ">=3.11.9" +requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.12'", "python_full_version < '3.12'", ] +[[package]] +name = "affine" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/98/d2f0bb06385069e799fc7d2870d9e078cfa0fa396dc8a2b81227d0da08b9/affine-2.4.0.tar.gz", hash = "sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea", size = 17132, upload-time = "2023-01-19T23:44:30.696Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/f7/85273299ab57117850cc0a936c64151171fac4da49bc6fba0dad984a7c5f/affine-2.4.0-py3-none-any.whl", hash = "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92", size = 15662, upload-time = "2023-01-19T23:44:28.833Z" }, +] + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -170,6 +179,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/ed/ad1755f82cd5a0baafe342e7154696a93e57f04f86515402f14e5beceb36/bump_my_version-1.2.7-py3-none-any.whl", hash = "sha256:16f89360f979c0a8eb3249ebe3e13ae4f0cb5481d7bb58e12a9f66996922acfd", size = 60013, upload-time = "2026-02-14T13:44:58.318Z" }, ] +[[package]] +name = "cartopy" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyproj" }, + { name = "pyshp" }, + { name = "shapely" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/3f/ec3dee34237b696a486d566a6d3ae6550ae821836e0412bafdcbbec2cfd2/cartopy-0.25.0.tar.gz", hash = "sha256:55f1a390e5f3f075b221c7d91fb10258ad978db786c7930eba06eb45d28753fe", size = 10767728, upload-time = "2025-08-01T12:44:16.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/e1/6a52ee21424da0ed30860f4e94d1657ade8d4436f0718485badf0e63011e/cartopy-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0e41d52160548a7ab7774423911db3bfb5a8bc0929580958b1945d3a004da872", size = 11006320, upload-time = "2025-08-01T12:43:48.13Z" }, + { url = "https://files.pythonhosted.org/packages/68/06/38bcfeab9822acffc86474659d33c4dc3c5dec4e61e9927fb8cc8617f651/cartopy-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:432e2a2688fc58af43b9b6bf1d343bb08e2d6ef298efa91e55445f1d308b5ef3", size = 10995635, upload-time = "2025-08-01T12:43:50.855Z" }, + { url = "https://files.pythonhosted.org/packages/a1/b6/f39407d27d641a949496a52ab00220fe0635758e3cb7afb4b7328abe17e7/cartopy-0.25.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:999e44021db07dcf895b115934fb0816aef39985fbaca6ded61d2536355531de", size = 11808214, upload-time = "2025-08-01T12:43:53.218Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c0/b33ac1f586608e80a5e10f3924e16c117da333fcb5e5240839e6681ac3d5/cartopy-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:4139e5ca9faaa037e0576cdcf625b9461a0b404d60e9d20ea24c4d8dbe6f689d", size = 10983301, upload-time = "2025-08-01T12:43:55.427Z" }, + { url = "https://files.pythonhosted.org/packages/63/35/b19901cbe7f1b118dccbb9e655cda7d01a31ee1ecd67e5d2d8afe119f6d3/cartopy-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:060a7b835c0c4222c1067b6ffb2f9c18458abaa35b6624573a3aa37ecf55f4bf", size = 11006900, upload-time = "2025-08-01T12:43:57.708Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4f/09e824f86be09152ec0f1fa1fe69affbd34eac7a13b545e2e08b9b6bc8ff/cartopy-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:57717cb603aecff03ecfee1bc153bb4022c054fcd51a4214a1bb53e5a6f74465", size = 10994813, upload-time = "2025-08-01T12:44:00.069Z" }, + { url = "https://files.pythonhosted.org/packages/b9/30/7465b650110514fc5c9c3b59935264c35ab56f876322de34efa55367ee4e/cartopy-0.25.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53c256351433155ef51dde976557212f4e230b8cca4e5d0d9b9a2737ad92959d", size = 11799069, upload-time = "2025-08-01T12:44:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/1d/52/3a57ecb4598c33ee06b512d3686e46b3983e65abd6ec94c5262d01930ed9/cartopy-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:efedb82f38409b72becdfee02231126952816d33a68b1c584bd2136713036bfb", size = 10983127, upload-time = "2025-08-01T12:44:04.441Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b9/0773ff8f1c755b8a362029e6910db87064d27ca021b060c48ce511ec98b7/cartopy-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a6fcd2df8039293096f957fc9c76e969b1a9715d12ab8cee1a6bdae0c6773b8b", size = 11007728, upload-time = "2025-08-01T12:44:06.64Z" }, + { url = "https://files.pythonhosted.org/packages/34/a6/75738630b7f64bca7afc6bc5de08ddf0c61f13563f2a1abf642373d1095e/cartopy-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4def451617e6957169447fe6ecdad0f63ef2d2007e7d451dd7b9656ada57382", size = 10996613, upload-time = "2025-08-01T12:44:08.822Z" }, + { url = "https://files.pythonhosted.org/packages/19/0d/669d4bbeb36b87ba504409d85c68ec297e6f434ea6525424f8aa5f14abac/cartopy-0.25.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c388824cb13e4fa9c2901dc4fbb2dbe9547acd2f4a6a3440983d4e6c6973ae3", size = 11829044, upload-time = "2025-08-01T12:44:11.402Z" }, + { url = "https://files.pythonhosted.org/packages/01/ff/b46e2120abd99b2ff3d376dc91ed58ae8f0a052d57c242c9b140497573dd/cartopy-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:60bad14c072d16e3c96967638cd66eb5a62cf24bc70087bcbfc6b30a3872ed26", size = 10987060, upload-time = "2025-08-01T12:44:14.222Z" }, +] + [[package]] name = "certifi" version = "2025.8.3" @@ -244,6 +281,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, ] +[[package]] +name = "click-plugins" +version = "1.1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, +] + +[[package]] +name = "cligj" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/0d/837dbd5d8430fd0f01ed72c4cfb2f548180f4c68c635df84ce87956cff32/cligj-0.7.2.tar.gz", hash = "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27", size = 9803, upload-time = "2021-05-28T21:23:27.935Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/86/43fa9f15c5b9fb6e82620428827cd3c284aa933431405d1bcf5231ae3d3e/cligj-0.7.2-py3-none-any.whl", hash = "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df", size = 7069, upload-time = "2021-05-28T21:23:26.877Z" }, +] + [[package]] name = "cloudpickle" version = "3.1.1" @@ -274,6 +335,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e3/51/9b208e85196941db2f0654ad0357ca6388ab3ed67efdbfc799f35d1f83aa/colorlog-6.9.0-py3-none-any.whl", hash = "sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff", size = 11424, upload-time = "2024-10-29T18:34:49.815Z" }, ] +[[package]] +name = "contextily" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "geopy" }, + { name = "joblib" }, + { name = "matplotlib" }, + { name = "mercantile" }, + { name = "pillow" }, + { name = "rasterio", version = "1.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "rasterio", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "requests" }, + { name = "xyzservices" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/f3/4c7fdb1ef92c8e3db1de9741595ae1815ddc8a7e2088539e6107c83e2268/contextily-1.7.0.tar.gz", hash = "sha256:6534faa5702b89b46d0d81b4c538754f2d8b3dd8cc298454b11ccedfa67e73ac", size = 22462157, upload-time = "2025-11-24T19:54:39.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/bb/6c824d0da874eef900ecf0a71e0889f0d4624560b0160fd9e333a146ee4f/contextily-1.7.0-py3-none-any.whl", hash = "sha256:38393b8e7dc38580ef1f60211a9ba1c3eb142a439c260509c201987754fa9dba", size = 16849, upload-time = "2025-11-24T19:54:37.018Z" }, +] + [[package]] name = "contourpy" version = "1.3.3" @@ -541,6 +622,44 @@ http = [ { name = "aiohttp" }, ] +[[package]] +name = "geographiclib" +version = "2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/78/4892343230a9d29faa1364564e525307a37e54ad776ea62c12129dbba704/geographiclib-2.1.tar.gz", hash = "sha256:6a6545e6262d0ed3522e13c515713718797e37ed8c672c31ad7b249f372ef108", size = 37004, upload-time = "2025-08-21T21:34:26Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b3/802576f2ea5dcb48501bb162e4c7b7b3ca5654a42b2c968ef98a797a4c79/geographiclib-2.1-py3-none-any.whl", hash = "sha256:e2a873b9b9e7fc38721ad73d5f4e6c9ed140d428a339970f505c07056997d40b", size = 40740, upload-time = "2025-08-21T21:34:24.955Z" }, +] + +[[package]] +name = "geopandas" +version = "1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyogrio" }, + { name = "pyproj" }, + { name = "shapely" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/ba/8e6b2091878e99e86a36a814dcaeff652ed48bdb03d53e78e15aaa63a914/geopandas-1.1.3.tar.gz", hash = "sha256:91a31989b6f566012838d21d5f8033f37dce882079ccb7cfdc40d5ccce7f284f", size = 336718, upload-time = "2026-03-09T21:49:09.545Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/78/6a04792ace63a93e162f1305392d500ae8ddcb620e7eb88a22fd622b35bb/geopandas-1.1.3-py3-none-any.whl", hash = "sha256:90d62a64f95eaa3be2ccc115c5f3d6e24208bb11983b390fdc0621a3eccd0230", size = 342514, upload-time = "2026-03-09T21:49:07.973Z" }, +] + +[[package]] +name = "geopy" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "geographiclib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/fd/ef6d53875ceab72c1fad22dbed5ec1ad04eb378c2251a6a8024bad890c3b/geopy-2.4.1.tar.gz", hash = "sha256:50283d8e7ad07d89be5cb027338c6365a32044df3ae2556ad3f52f4840b3d0d1", size = 117625, upload-time = "2023-11-23T21:49:32.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/15/cf2a69ade4b194aa524ac75112d5caac37414b20a3a03e6865dfe0bd1539/geopy-2.4.1-py3-none-any.whl", hash = "sha256:ae8b4bc5c1131820f4d75fce9d4aaaca0c85189b3aa5d64c3dcaf5e3b7b882a7", size = 125437, upload-time = "2023-11-23T21:49:30.421Z" }, +] + [[package]] name = "greenlet" version = "3.2.4" @@ -1000,6 +1119,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/53/8d8fa0ea32a8c8239e04d022f6c059ee5e1b77517769feccd50f1df43d6d/matplotlib-3.10.6-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d6ca6ef03dfd269f4ead566ec6f3fb9becf8dab146fb999022ed85ee9f6b3eb", size = 8693933, upload-time = "2025-08-30T00:14:22.942Z" }, ] +[[package]] +name = "matplotlib-scalebar" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/3d/e76e0e48c96e43b656446c35990824f20136de21940a7436020ab757a54d/matplotlib_scalebar-0.9.0.tar.gz", hash = "sha256:f91adc5e4e67ac920a8fa98a55f904d56e5c2da4d56286748c31ec62146362be", size = 1180588, upload-time = "2025-01-17T22:10:59.349Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/c0/2dfab7b319dabe23f5a7b515a797c74b501d15c72e7a03837cf0cf779b9e/matplotlib_scalebar-0.9.0-py3-none-any.whl", hash = "sha256:5140525cd4e0c60bcade541b86571dabaf446fa69192530bd82d60b54601aa79", size = 16411, upload-time = "2025-01-17T22:10:57.319Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -1009,6 +1140,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mercantile" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/c6/87409bcb26fb68c393fa8cf58ba09363aa7298cfb438a0109b5cb14bc98b/mercantile-1.2.1.tar.gz", hash = "sha256:fa3c6db15daffd58454ac198b31887519a19caccee3f9d63d17ae7ff61b3b56b", size = 26352, upload-time = "2021-04-21T14:42:41.096Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/d6/de0cc74f8d36976aeca0dd2e9cbf711882ff8e177495115fd82459afdc4d/mercantile-1.2.1-py3-none-any.whl", hash = "sha256:30f457a73ee88261aab787b7069d85961a5703bb09dc57a170190bc042cd023f", size = 14779, upload-time = "2021-04-21T14:42:39.841Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -1876,6 +2019,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/38/dee2fc597010654c3a7588f0990192680c5f2d405fbf6f2e5f5c4df1c736/pyod-2.0.5-py3-none-any.whl", hash = "sha256:1a3182529c68e388be0671840bb11f83d5d1deb738ad931d294881f7d50e4fdb", size = 200628, upload-time = "2025-04-29T22:26:57.284Z" }, ] +[[package]] +name = "pyogrio" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "numpy" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/d4/12f86b1ed09721363da4c09622464b604c851a9223fc0c6b393fb2012208/pyogrio-0.12.1.tar.gz", hash = "sha256:e548ab705bb3e5383693717de1e6c76da97f3762ab92522cb310f93128a75ff1", size = 303289, upload-time = "2025-11-28T19:04:53.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/46/b2c2dcdfd88759b56f103365905fffb85e8b08c1db1ec7c8f8b4c4c26016/pyogrio-0.12.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:01b322dac2a258d24b024d1028dcaa03c9bb6d9c3988b86d298a64873d10dc65", size = 23670744, upload-time = "2025-11-28T19:03:11.299Z" }, + { url = "https://files.pythonhosted.org/packages/d9/21/b69f1bc51d805c00dd7c484a18e1fd2e75b41da1d9f5b8591d7d9d4a7d2f/pyogrio-0.12.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:e10087abcbd6b7e8212560a7002984e5078ac7b3a969ddc2c9929044dbb0d403", size = 25246184, upload-time = "2025-11-28T19:03:13.997Z" }, + { url = "https://files.pythonhosted.org/packages/19/8c/b6aae08e8fcc4f2a903da5f6bd8f888d2b6d7290e54dde5abe15b4cca8df/pyogrio-0.12.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1f6c621972b09fd81a32317e742c69ff4a7763a803da211361a78317f9577765", size = 31434449, upload-time = "2025-11-28T19:03:16.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/f9/9538fa893c29a3fdfeddf3b4c9f8db77f2d4134bc766587929fec8405ebf/pyogrio-0.12.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c38253427b688464caad5316d4ebcec116b5e13f1f02cc4e3588502f136ca1b4", size = 30987586, upload-time = "2025-11-28T19:03:19.586Z" }, + { url = "https://files.pythonhosted.org/packages/89/a4/0aef5837b4e11840f501e48e01c31242838476c4f4aff9c05e228a083982/pyogrio-0.12.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5f47787251de7ce13cc06038da93a1214dc283cbccf816be6e03c080358226c8", size = 32534386, upload-time = "2025-11-28T19:03:22.292Z" }, + { url = "https://files.pythonhosted.org/packages/34/97/e8f2ed8a339152b86f8403c258ae5d5f23ab32d690eeb0545bb3473d0c69/pyogrio-0.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:c1d756cf2da4cdf5609779f260d1e1e89be023184225855d6f3dcd33bbe17cb0", size = 22941718, upload-time = "2025-11-28T19:03:24.82Z" }, + { url = "https://files.pythonhosted.org/packages/ad/e0/656b6536549d41b5aec57e0deca1f269b4f17532f0636836f587e581603a/pyogrio-0.12.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:7a0d5ca39184030aec4cde30f4258f75b227a854530d2659babc8189d76e657d", size = 23661857, upload-time = "2025-11-28T19:03:27.744Z" }, + { url = "https://files.pythonhosted.org/packages/14/78/313259e40da728bdb60106ffdc7ea8224d164498cb838ecb79b634aab967/pyogrio-0.12.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:feaff42bbe8087ca0b30e33b09d1ce049ca55fe83ad83db1139ef37d1d04f30c", size = 25237106, upload-time = "2025-11-28T19:03:30.018Z" }, + { url = "https://files.pythonhosted.org/packages/8f/ca/5368571a8b00b941ccfbe6ea29a5566aaffd45d4eb1553b956f7755af43e/pyogrio-0.12.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81096a5139532de5a8003ef02b41d5d2444cb382a9aecd1165b447eb549180d3", size = 31417048, upload-time = "2025-11-28T19:03:32.572Z" }, + { url = "https://files.pythonhosted.org/packages/ef/85/6eeb875f27bf498d657eb5dab9f58e4c48b36c9037122787abee9a1ba4ba/pyogrio-0.12.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:41b78863f782f7a113ed0d36a5dc74d59735bd3a82af53510899bb02a18b06bb", size = 30952115, upload-time = "2025-11-28T19:03:35.332Z" }, + { url = "https://files.pythonhosted.org/packages/36/f7/cf8bec9024625947e1a71441906f60a5fa6f9e4c441c4428037e73b1fcc8/pyogrio-0.12.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:8b65be8c4258b27cc8f919b21929cecdadda4c353e3637fa30850339ef4d15c5", size = 32537246, upload-time = "2025-11-28T19:03:37.969Z" }, + { url = "https://files.pythonhosted.org/packages/ab/10/7c9f5e428273574e69f217eba3a6c0c42936188ad4dcd9e2c41ebb711188/pyogrio-0.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:1291b866c2c81d991bda15021b08b3621709b40ee3a85689229929e9465788bf", size = 22933980, upload-time = "2025-11-28T19:03:41.047Z" }, + { url = "https://files.pythonhosted.org/packages/be/56/f56e79f71b84aa9bea25fdde39fab3846841bd7926be96f623eb7253b7e1/pyogrio-0.12.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:ec0e47a5a704e575092b2fd5c83fa0472a1d421e590f94093eb837bb0a11125d", size = 23658483, upload-time = "2025-11-28T19:03:43.567Z" }, + { url = "https://files.pythonhosted.org/packages/66/ac/5559f8a35d58a16cbb2dd7602dd11936ff8796d8c9bf789f14da88764ec3/pyogrio-0.12.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:b4c888fc08f388be4dd99dfca5e84a5cdc5994deeec0230cc45144d3460e2b21", size = 25232737, upload-time = "2025-11-28T19:03:45.92Z" }, + { url = "https://files.pythonhosted.org/packages/59/58/925f1c129ddd7cbba8dea4e7609797cea7a76dbc863ac9afd318a679c4b9/pyogrio-0.12.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:73a88436f9962750d782853727897ac2722cac5900d920e39fab3e56d7a6a7f1", size = 31377986, upload-time = "2025-11-28T19:03:48.495Z" }, + { url = "https://files.pythonhosted.org/packages/18/5f/c87034e92847b1844d0e8492a6a8e3301147d32c5e57909397ce64dbedf5/pyogrio-0.12.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b5d248a0d59fe9bbf9a35690b70004c67830ee0ebe7d4f7bb8ffd8659f684b3a", size = 30915791, upload-time = "2025-11-28T19:03:51.267Z" }, + { url = "https://files.pythonhosted.org/packages/46/35/b874f79d03e9f900012cf609f7fff97b77164f2e14ee5aac282f8a999c1b/pyogrio-0.12.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0622bc1a186421547660271083079b38d42e6f868802936d8538c0b379f1ab6b", size = 32499754, upload-time = "2025-11-28T19:03:58.776Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c4/705678c9c4200130290b3a104b45c0cc10aaa48fcef3b2585b34e34ab3e1/pyogrio-0.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:207bd60c7ffbcea84584596e3637653aa7095e9ee20fa408f90c7f9460392613", size = 22933945, upload-time = "2025-11-28T19:04:01.551Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e0/d92d4944001330bc87742d43f112d63d12fc89378b6187e62ff3fc1e8e85/pyogrio-0.12.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1511b39a283fa27cda906cd187a791578942a87a40b6a06697d9b43bb8ac80b0", size = 23692697, upload-time = "2025-11-28T19:04:04.208Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d7/40acbe06d1b1140e3bb27b79e9163776469c1dc785f1be7d9a7fc7b95c87/pyogrio-0.12.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:e486cd6aa9ea8a15394a5f84e019d61ec18f257eeeb642348bd68c3d1e57280b", size = 25258083, upload-time = "2025-11-28T19:04:07.121Z" }, + { url = "https://files.pythonhosted.org/packages/87/a1/39fefd9cddd95986700524f43d3093b4350f6e4fc200623c3838424a5080/pyogrio-0.12.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d3f1a19f63bfd1d3042e45f37ad1d6598123a5a604b6c4ba3f38b419273486cd", size = 31368995, upload-time = "2025-11-28T19:04:09.88Z" }, + { url = "https://files.pythonhosted.org/packages/18/d7/da88c566e67d741a03851eb8d01358949d52e0b0fc2cd953582dc6d89ff8/pyogrio-0.12.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:f3dcc59b3316b8a0f59346bcc638a4d69997864a4d21da839192f50c4c92369a", size = 31035589, upload-time = "2025-11-28T19:04:12.993Z" }, + { url = "https://files.pythonhosted.org/packages/11/ac/8f0199f0d31b8ddbc4b4ea1918df8070fdf3e0a63100b898633ec9396224/pyogrio-0.12.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a0643e041dee3e8e038fce69f52a915ecb486e6d7b674c0f9919f3c9e9629689", size = 32487973, upload-time = "2025-11-28T19:04:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/bd/64/8541a27e9635a335835d234dfaeb19d6c26097fd88224eda7791f83ca98d/pyogrio-0.12.1-cp313-cp313t-win_amd64.whl", hash = "sha256:5881017f29e110d3613819667657844d8e961b747f2d35cf92f273c27af6d068", size = 22987374, upload-time = "2025-11-28T19:04:18.91Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6f/b4d5e285e08c0c60bcc23b50d73038ddc7335d8de79cc25678cd486a3db0/pyogrio-0.12.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5a1b0453d1c9e7b03715dd57296c8f3790acb8b50d7e3b5844b3074a18f50709", size = 23660673, upload-time = "2025-11-28T19:04:21.662Z" }, + { url = "https://files.pythonhosted.org/packages/8d/75/4b29e71489c5551aa1a1c5ca8c5160a60203c94f2f68c87c0e3614d58965/pyogrio-0.12.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e7ee560422239dd09ca7f8284cc8483a8919c30d25f3049bb0249bff4c38dec4", size = 25232194, upload-time = "2025-11-28T19:04:23.975Z" }, + { url = "https://files.pythonhosted.org/packages/89/6e/e9929d2261a07c36301983de2767bcde90d441ab5bf1d767ce56dd07f8b4/pyogrio-0.12.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:648c6f7f5f214d30e6cf493b4af1d59782907ac068af9119ca35f18153d6865a", size = 31336936, upload-time = "2025-11-28T19:04:26.594Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9e/c59941d734ed936d4e5c89b4b99cb5541307cc42b3fd466ee78a1850c177/pyogrio-0.12.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:58042584f3fd4cabb0f55d26c1405053f656be8a5c266c38140316a1e981aca0", size = 30902210, upload-time = "2025-11-28T19:04:29.143Z" }, + { url = "https://files.pythonhosted.org/packages/d1/68/cc07320a63f9c2586e60bf11d148b00e12d0e707673bffe609bbdcb7e754/pyogrio-0.12.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b438e38e4ccbaedaa5cb5824ff5de5539315d9b2fde6547c1e816576924ee8ca", size = 32461674, upload-time = "2025-11-28T19:04:31.792Z" }, + { url = "https://files.pythonhosted.org/packages/13/bc/e4522f429c45a3b6ad28185849dd76e5c8718b780883c4795e7ee41841ae/pyogrio-0.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:f1d8d8a2fea3781dc2a05982c050259261ebc0f6c5e03732d6d79d582adf9363", size = 23550575, upload-time = "2025-11-28T19:04:34.556Z" }, + { url = "https://files.pythonhosted.org/packages/bd/ac/34f0664d0e391994a7b68529ae07a96432b2b4926dbac173ddc4ec94d310/pyogrio-0.12.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:9fe7286946f35a73e6370dc5855bc7a5e8e7babf9e4a8bad7a3279a1d94c7ea9", size = 23694285, upload-time = "2025-11-28T19:04:37.833Z" }, + { url = "https://files.pythonhosted.org/packages/8a/93/873255529faff1da09d0b27287e85ec805a318c60c0c74fd7df77f94e557/pyogrio-0.12.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2c50345b382f1be801d654ec22c70ee974d6057d4ba7afe984b55f2192bc94ee", size = 25259825, upload-time = "2025-11-28T19:04:40.125Z" }, + { url = "https://files.pythonhosted.org/packages/27/95/4d4c3644695d99c6fa0b0b42f0d6266ae9dfaf64478a3371eaac950bdd02/pyogrio-0.12.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0db95765ac0ca935c7fe579e29451294e3ab19c317b0c59c31fbe92a69155e0", size = 31371995, upload-time = "2025-11-28T19:04:42.736Z" }, + { url = "https://files.pythonhosted.org/packages/4c/6f/71f6bcca8754c8bf55a4b7153c61c91f8ac5ba992568e9fa3e54a0ee76fd/pyogrio-0.12.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fc882779075982b93064b3bf3d8642514a6df00d9dd752493b104817072cfb01", size = 31035498, upload-time = "2025-11-28T19:04:45.79Z" }, + { url = "https://files.pythonhosted.org/packages/fd/47/75c1aa165a988347317afab9b938a01ad25dbca559b582ea34473703dc38/pyogrio-0.12.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:806f620e0c54b54dbdd65e9b6368d24f344cda84c9343364b40a57eb3e1c4dca", size = 32496390, upload-time = "2025-11-28T19:04:48.786Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/4641dc5d952f6bdb71dabad2c50e3f8a5d58396cdea6ff8f8a08bfd4f4a6/pyogrio-0.12.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5399f66730978d8852ef5f44dbafa0f738e7f28f4f784349f36830b69a9d2134", size = 23620996, upload-time = "2025-11-28T19:04:51.132Z" }, +] + [[package]] name = "pyparsing" version = "3.2.5" @@ -1885,6 +2077,80 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl", hash = "sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e", size = 113890, upload-time = "2025-09-21T04:11:04.117Z" }, ] +[[package]] +name = "pyproj" +version = "3.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/90/67bd7260b4ea9b8b20b4f58afef6c223ecb3abf368eb4ec5bc2cdef81b49/pyproj-3.7.2.tar.gz", hash = "sha256:39a0cf1ecc7e282d1d30f36594ebd55c9fae1fda8a2622cee5d100430628f88c", size = 226279, upload-time = "2025-08-14T12:05:42.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/bd/f205552cd1713b08f93b09e39a3ec99edef0b3ebbbca67b486fdf1abe2de/pyproj-3.7.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:2514d61f24c4e0bb9913e2c51487ecdaeca5f8748d8313c933693416ca41d4d5", size = 6227022, upload-time = "2025-08-14T12:03:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/75/4c/9a937e659b8b418ab573c6d340d27e68716928953273e0837e7922fcac34/pyproj-3.7.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:8693ca3892d82e70de077701ee76dd13d7bca4ae1c9d1e739d72004df015923a", size = 4625810, upload-time = "2025-08-14T12:03:53.808Z" }, + { url = "https://files.pythonhosted.org/packages/c0/7d/a9f41e814dc4d1dc54e95b2ccaf0b3ebe3eb18b1740df05fe334724c3d89/pyproj-3.7.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5e26484d80fea56273ed1555abaea161e9661d81a6c07815d54b8e883d4ceb25", size = 9638694, upload-time = "2025-08-14T12:03:55.669Z" }, + { url = "https://files.pythonhosted.org/packages/ad/ab/9bdb4a6216b712a1f9aab1c0fcbee5d3726f34a366f29c3e8c08a78d6b70/pyproj-3.7.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:281cb92847814e8018010c48b4069ff858a30236638631c1a91dd7bfa68f8a8a", size = 9493977, upload-time = "2025-08-14T12:03:57.937Z" }, + { url = "https://files.pythonhosted.org/packages/c9/db/2db75b1b6190f1137b1c4e8ef6a22e1c338e46320f6329bfac819143e063/pyproj-3.7.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9c8577f0b7bb09118ec2e57e3babdc977127dd66326d6c5d755c76b063e6d9dc", size = 10841151, upload-time = "2025-08-14T12:04:00.271Z" }, + { url = "https://files.pythonhosted.org/packages/89/f7/989643394ba23a286e9b7b3f09981496172f9e0d4512457ffea7dc47ffc7/pyproj-3.7.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a23f59904fac3a5e7364b3aa44d288234af267ca041adb2c2b14a903cd5d3ac5", size = 10751585, upload-time = "2025-08-14T12:04:02.228Z" }, + { url = "https://files.pythonhosted.org/packages/53/6d/ad928fe975a6c14a093c92e6a319ca18f479f3336bb353a740bdba335681/pyproj-3.7.2-cp311-cp311-win32.whl", hash = "sha256:f2af4ed34b2cf3e031a2d85b067a3ecbd38df073c567e04b52fa7a0202afde8a", size = 5908533, upload-time = "2025-08-14T12:04:04.821Z" }, + { url = "https://files.pythonhosted.org/packages/79/e0/b95584605cec9ed50b7ebaf7975d1c4ddeec5a86b7a20554ed8b60042bd7/pyproj-3.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:0b7cb633565129677b2a183c4d807c727d1c736fcb0568a12299383056e67433", size = 6320742, upload-time = "2025-08-14T12:04:06.357Z" }, + { url = "https://files.pythonhosted.org/packages/b7/4d/536e8f93bca808175c2d0a5ac9fdf69b960d8ab6b14f25030dccb07464d7/pyproj-3.7.2-cp311-cp311-win_arm64.whl", hash = "sha256:38b08d85e3a38e455625b80e9eb9f78027c8e2649a21dec4df1f9c3525460c71", size = 6245772, upload-time = "2025-08-14T12:04:08.365Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ab/9893ea9fb066be70ed9074ae543914a618c131ed8dff2da1e08b3a4df4db/pyproj-3.7.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:0a9bb26a6356fb5b033433a6d1b4542158fb71e3c51de49b4c318a1dff3aeaab", size = 6219832, upload-time = "2025-08-14T12:04:10.264Z" }, + { url = "https://files.pythonhosted.org/packages/53/78/4c64199146eed7184eb0e85bedec60a4aa8853b6ffe1ab1f3a8b962e70a0/pyproj-3.7.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:567caa03021178861fad27fabde87500ec6d2ee173dd32f3e2d9871e40eebd68", size = 4620650, upload-time = "2025-08-14T12:04:11.978Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ac/14a78d17943898a93ef4f8c6a9d4169911c994e3161e54a7cedeba9d8dde/pyproj-3.7.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c203101d1dc3c038a56cff0447acc515dd29d6e14811406ac539c21eed422b2a", size = 9667087, upload-time = "2025-08-14T12:04:13.964Z" }, + { url = "https://files.pythonhosted.org/packages/b8/be/212882c450bba74fc8d7d35cbd57e4af84792f0a56194819d98106b075af/pyproj-3.7.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1edc34266c0c23ced85f95a1ee8b47c9035eae6aca5b6b340327250e8e281630", size = 9552797, upload-time = "2025-08-14T12:04:16.624Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c0/c0f25c87b5d2a8686341c53c1792a222a480d6c9caf60311fec12c99ec26/pyproj-3.7.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa9f26c21bc0e2dc3d224cb1eb4020cf23e76af179a7c66fea49b828611e4260", size = 10837036, upload-time = "2025-08-14T12:04:18.733Z" }, + { url = "https://files.pythonhosted.org/packages/5d/37/5cbd6772addde2090c91113332623a86e8c7d583eccb2ad02ea634c4a89f/pyproj-3.7.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9428b318530625cb389b9ddc9c51251e172808a4af79b82809376daaeabe5e9", size = 10775952, upload-time = "2025-08-14T12:04:20.709Z" }, + { url = "https://files.pythonhosted.org/packages/69/a1/dc250e3cf83eb4b3b9a2cf86fdb5e25288bd40037ae449695550f9e96b2f/pyproj-3.7.2-cp312-cp312-win32.whl", hash = "sha256:b3d99ed57d319da042f175f4554fc7038aa4bcecc4ac89e217e350346b742c9d", size = 5898872, upload-time = "2025-08-14T12:04:22.485Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a6/6fe724b72b70f2b00152d77282e14964d60ab092ec225e67c196c9b463e5/pyproj-3.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:11614a054cd86a2ed968a657d00987a86eeb91fdcbd9ad3310478685dc14a128", size = 6312176, upload-time = "2025-08-14T12:04:24.736Z" }, + { url = "https://files.pythonhosted.org/packages/5d/68/915cc32c02a91e76d02c8f55d5a138d6ef9e47a0d96d259df98f4842e558/pyproj-3.7.2-cp312-cp312-win_arm64.whl", hash = "sha256:509a146d1398bafe4f53273398c3bb0b4732535065fa995270e52a9d3676bca3", size = 6233452, upload-time = "2025-08-14T12:04:27.287Z" }, + { url = "https://files.pythonhosted.org/packages/be/14/faf1b90d267cea68d7e70662e7f88cefdb1bc890bd596c74b959e0517a72/pyproj-3.7.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:19466e529b1b15eeefdf8ff26b06fa745856c044f2f77bf0edbae94078c1dfa1", size = 6214580, upload-time = "2025-08-14T12:04:28.804Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/da9a45b184d375f62667f62eba0ca68569b0bd980a0bb7ffcc1d50440520/pyproj-3.7.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:c79b9b84c4a626c5dc324c0d666be0bfcebd99f7538d66e8898c2444221b3da7", size = 4615388, upload-time = "2025-08-14T12:04:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e7/d2b459a4a64bca328b712c1b544e109df88e5c800f7c143cfbc404d39bfb/pyproj-3.7.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ceecf374cacca317bc09e165db38ac548ee3cad07c3609442bd70311c59c21aa", size = 9628455, upload-time = "2025-08-14T12:04:32.435Z" }, + { url = "https://files.pythonhosted.org/packages/f8/85/c2b1706e51942de19076eff082f8495e57d5151364e78b5bef4af4a1d94a/pyproj-3.7.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5141a538ffdbe4bfd157421828bb2e07123a90a7a2d6f30fa1462abcfb5ce681", size = 9514269, upload-time = "2025-08-14T12:04:34.599Z" }, + { url = "https://files.pythonhosted.org/packages/34/38/07a9b89ae7467872f9a476883a5bad9e4f4d1219d31060f0f2b282276cbe/pyproj-3.7.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f000841e98ea99acbb7b8ca168d67773b0191de95187228a16110245c5d954d5", size = 10808437, upload-time = "2025-08-14T12:04:36.485Z" }, + { url = "https://files.pythonhosted.org/packages/12/56/fda1daeabbd39dec5b07f67233d09f31facb762587b498e6fc4572be9837/pyproj-3.7.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8115faf2597f281a42ab608ceac346b4eb1383d3b45ab474fd37341c4bf82a67", size = 10745540, upload-time = "2025-08-14T12:04:38.568Z" }, + { url = "https://files.pythonhosted.org/packages/0d/90/c793182cbba65a39a11db2ac6b479fe76c59e6509ae75e5744c344a0da9d/pyproj-3.7.2-cp313-cp313-win32.whl", hash = "sha256:f18c0579dd6be00b970cb1a6719197fceecc407515bab37da0066f0184aafdf3", size = 5896506, upload-time = "2025-08-14T12:04:41.059Z" }, + { url = "https://files.pythonhosted.org/packages/be/0f/747974129cf0d800906f81cd25efd098c96509026e454d4b66868779ab04/pyproj-3.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:bb41c29d5f60854b1075853fe80c58950b398d4ebb404eb532536ac8d2834ed7", size = 6310195, upload-time = "2025-08-14T12:04:42.974Z" }, + { url = "https://files.pythonhosted.org/packages/82/64/fc7598a53172c4931ec6edf5228280663063150625d3f6423b4c20f9daff/pyproj-3.7.2-cp313-cp313-win_arm64.whl", hash = "sha256:2b617d573be4118c11cd96b8891a0b7f65778fa7733ed8ecdb297a447d439100", size = 6230748, upload-time = "2025-08-14T12:04:44.491Z" }, + { url = "https://files.pythonhosted.org/packages/aa/f0/611dd5cddb0d277f94b7af12981f56e1441bf8d22695065d4f0df5218498/pyproj-3.7.2-cp313-cp313t-macosx_13_0_x86_64.whl", hash = "sha256:d27b48f0e81beeaa2b4d60c516c3a1cfbb0c7ff6ef71256d8e9c07792f735279", size = 6241729, upload-time = "2025-08-14T12:04:46.274Z" }, + { url = "https://files.pythonhosted.org/packages/15/93/40bd4a6c523ff9965e480870611aed7eda5aa2c6128c6537345a2b77b542/pyproj-3.7.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:55a3610d75023c7b1c6e583e48ef8f62918e85a2ae81300569d9f104d6684bb6", size = 4652497, upload-time = "2025-08-14T12:04:48.203Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ae/7150ead53c117880b35e0d37960d3138fe640a235feb9605cb9386f50bb0/pyproj-3.7.2-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:8d7349182fa622696787cc9e195508d2a41a64765da9b8a6bee846702b9e6220", size = 9942610, upload-time = "2025-08-14T12:04:49.652Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/7a4a7eafecf2b46ab64e5c08176c20ceb5844b503eaa551bf12ccac77322/pyproj-3.7.2-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:d230b186eb876ed4f29a7c5ee310144c3a0e44e89e55f65fb3607e13f6db337c", size = 9692390, upload-time = "2025-08-14T12:04:51.731Z" }, + { url = "https://files.pythonhosted.org/packages/c3/55/ae18f040f6410f0ea547a21ada7ef3e26e6c82befa125b303b02759c0e9d/pyproj-3.7.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:237499c7862c578d0369e2b8ac56eec550e391a025ff70e2af8417139dabb41c", size = 11047596, upload-time = "2025-08-14T12:04:53.748Z" }, + { url = "https://files.pythonhosted.org/packages/e6/2e/d3fff4d2909473f26ae799f9dda04caa322c417a51ff3b25763f7d03b233/pyproj-3.7.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8c225f5978abd506fd9a78eaaf794435e823c9156091cabaab5374efb29d7f69", size = 10896975, upload-time = "2025-08-14T12:04:55.875Z" }, + { url = "https://files.pythonhosted.org/packages/f2/bc/8fc7d3963d87057b7b51ebe68c1e7c51c23129eee5072ba6b86558544a46/pyproj-3.7.2-cp313-cp313t-win32.whl", hash = "sha256:2da731876d27639ff9d2d81c151f6ab90a1546455fabd93368e753047be344a2", size = 5953057, upload-time = "2025-08-14T12:04:58.466Z" }, + { url = "https://files.pythonhosted.org/packages/cc/27/ea9809966cc47d2d51e6d5ae631ea895f7c7c7b9b3c29718f900a8f7d197/pyproj-3.7.2-cp313-cp313t-win_amd64.whl", hash = "sha256:f54d91ae18dd23b6c0ab48126d446820e725419da10617d86a1b69ada6d881d3", size = 6375414, upload-time = "2025-08-14T12:04:59.861Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f8/1ef0129fba9a555c658e22af68989f35e7ba7b9136f25758809efec0cd6e/pyproj-3.7.2-cp313-cp313t-win_arm64.whl", hash = "sha256:fc52ba896cfc3214dc9f9ca3c0677a623e8fdd096b257c14a31e719d21ff3fdd", size = 6262501, upload-time = "2025-08-14T12:05:01.39Z" }, + { url = "https://files.pythonhosted.org/packages/42/17/c2b050d3f5b71b6edd0d96ae16c990fdc42a5f1366464a5c2772146de33a/pyproj-3.7.2-cp314-cp314-macosx_13_0_x86_64.whl", hash = "sha256:2aaa328605ace41db050d06bac1adc11f01b71fe95c18661497763116c3a0f02", size = 6214541, upload-time = "2025-08-14T12:05:03.166Z" }, + { url = "https://files.pythonhosted.org/packages/03/68/68ada9c8aea96ded09a66cfd9bf87aa6db8c2edebe93f5bf9b66b0143fbc/pyproj-3.7.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:35dccbce8201313c596a970fde90e33605248b66272595c061b511c8100ccc08", size = 4617456, upload-time = "2025-08-14T12:05:04.563Z" }, + { url = "https://files.pythonhosted.org/packages/81/e4/4c50ceca7d0e937977866b02cb64e6ccf4df979a5871e521f9e255df6073/pyproj-3.7.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:25b0b7cb0042444c29a164b993c45c1b8013d6c48baa61dc1160d834a277e83b", size = 9615590, upload-time = "2025-08-14T12:05:06.094Z" }, + { url = "https://files.pythonhosted.org/packages/05/1e/ada6fb15a1d75b5bd9b554355a69a798c55a7dcc93b8d41596265c1772e3/pyproj-3.7.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:85def3a6388e9ba51f964619aa002a9d2098e77c6454ff47773bb68871024281", size = 9474960, upload-time = "2025-08-14T12:05:07.973Z" }, + { url = "https://files.pythonhosted.org/packages/51/07/9d48ad0a8db36e16f842f2c8a694c1d9d7dcf9137264846bef77585a71f3/pyproj-3.7.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b1bccefec3875ab81eabf49059e2b2ea77362c178b66fd3528c3e4df242f1516", size = 10799478, upload-time = "2025-08-14T12:05:14.102Z" }, + { url = "https://files.pythonhosted.org/packages/85/cf/2f812b529079f72f51ff2d6456b7fef06c01735e5cfd62d54ffb2b548028/pyproj-3.7.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d5371ca114d6990b675247355a801925814eca53e6c4b2f1b5c0a956336ee36e", size = 10710030, upload-time = "2025-08-14T12:05:16.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/9b/4626a19e1f03eba4c0e77b91a6cf0f73aa9cb5d51a22ee385c22812bcc2c/pyproj-3.7.2-cp314-cp314-win32.whl", hash = "sha256:77f066626030f41be543274f5ac79f2a511fe89860ecd0914f22131b40a0ec25", size = 5991181, upload-time = "2025-08-14T12:05:19.492Z" }, + { url = "https://files.pythonhosted.org/packages/04/b2/5a6610554306a83a563080c2cf2c57565563eadd280e15388efa00fb5b33/pyproj-3.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:5a964da1696b8522806f4276ab04ccfff8f9eb95133a92a25900697609d40112", size = 6434721, upload-time = "2025-08-14T12:05:21.022Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ce/6c910ea2e1c74ef673c5d48c482564b8a7824a44c4e35cca2e765b68cfcc/pyproj-3.7.2-cp314-cp314-win_arm64.whl", hash = "sha256:e258ab4dbd3cf627809067c0ba8f9884ea76c8e5999d039fb37a1619c6c3e1f6", size = 6363821, upload-time = "2025-08-14T12:05:22.627Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e4/5532f6f7491812ba782a2177fe9de73fd8e2912b59f46a1d056b84b9b8f2/pyproj-3.7.2-cp314-cp314t-macosx_13_0_x86_64.whl", hash = "sha256:bbbac2f930c6d266f70ec75df35ef851d96fdb3701c674f42fd23a9314573b37", size = 6241773, upload-time = "2025-08-14T12:05:24.577Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/0938c3f2bbbef1789132d1726d9b0e662f10cfc22522743937f421ad664e/pyproj-3.7.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:b7544e0a3d6339dc9151e9c8f3ea62a936ab7cc446a806ec448bbe86aebb979b", size = 4652537, upload-time = "2025-08-14T12:05:26.391Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a8/488b1ed47d25972f33874f91f09ca8f2227902f05f63a2b80dc73e7b1c97/pyproj-3.7.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f7f5133dca4c703e8acadf6f30bc567d39a42c6af321e7f81975c2518f3ed357", size = 9940864, upload-time = "2025-08-14T12:05:27.985Z" }, + { url = "https://files.pythonhosted.org/packages/c7/cc/7f4c895d0cb98e47b6a85a6d79eaca03eb266129eed2f845125c09cf31ff/pyproj-3.7.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5aff3343038d7426aa5076f07feb88065f50e0502d1b0d7c22ddfdd2c75a3f81", size = 9688868, upload-time = "2025-08-14T12:05:30.425Z" }, + { url = "https://files.pythonhosted.org/packages/b2/b7/c7e306b8bb0f071d9825b753ee4920f066c40fbfcce9372c4f3cfb2fc4ed/pyproj-3.7.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b0552178c61f2ac1c820d087e8ba6e62b29442debddbb09d51c4bf8acc84d888", size = 11045910, upload-time = "2025-08-14T12:05:32.507Z" }, + { url = "https://files.pythonhosted.org/packages/42/fb/538a4d2df695980e2dde5c04d965fbdd1fe8c20a3194dc4aaa3952a4d1be/pyproj-3.7.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:47d87db2d2c436c5fd0409b34d70bb6cdb875cca2ebe7a9d1c442367b0ab8d59", size = 10895724, upload-time = "2025-08-14T12:05:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/e8/8b/a3f0618b03957de9db5489a04558a8826f43906628bb0b766033aa3b5548/pyproj-3.7.2-cp314-cp314t-win32.whl", hash = "sha256:c9b6f1d8ad3e80a0ee0903a778b6ece7dca1d1d40f6d114ae01bc8ddbad971aa", size = 6056848, upload-time = "2025-08-14T12:05:37.553Z" }, + { url = "https://files.pythonhosted.org/packages/bc/56/413240dd5149dd3291eda55aa55a659da4431244a2fd1319d0ae89407cfb/pyproj-3.7.2-cp314-cp314t-win_amd64.whl", hash = "sha256:1914e29e27933ba6f9822663ee0600f169014a2859f851c054c88cf5ea8a333c", size = 6517676, upload-time = "2025-08-14T12:05:39.126Z" }, + { url = "https://files.pythonhosted.org/packages/15/73/a7141a1a0559bf1a7aa42a11c879ceb19f02f5c6c371c6d57fd86cefd4d1/pyproj-3.7.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d9d25bae416a24397e0d85739f84d323b55f6511e45a522dd7d7eae70d10c7e4", size = 6391844, upload-time = "2025-08-14T12:05:40.745Z" }, +] + +[[package]] +name = "pyshp" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/20/8b07bae73aaa0c3f5a2683ba6e23b46e977e2d33a88126d56bbcc2d135cd/pyshp-3.0.3.tar.gz", hash = "sha256:bf4678b13dd53578ed87669676a2fffeccbcded1ec8ff9cafb36d1b660f4b305", size = 2192568, upload-time = "2025-11-28T17:47:31.616Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/06/cad54e8ce758bd836ee5411691cbd49efeb9cc611b374670fce299519334/pyshp-3.0.3-py3-none-any.whl", hash = "sha256:28c8fac8c0c25bb0fecbbfd10ead7f319c2ff2f3b0b44a94f22bd2c93510ad42", size = 58465, upload-time = "2025-11-28T17:47:30.328Z" }, +] + [[package]] name = "pytest" version = "8.4.2" @@ -2017,6 +2283,113 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/26/1062c7ec1b053db9e499b4d2d5bc231743201b74051c973dadeac80a8f43/questionary-2.1.1-py3-none-any.whl", hash = "sha256:a51af13f345f1cdea62347589fbb6df3b290306ab8930713bfae4d475a7d4a59", size = 36753, upload-time = "2025-08-28T19:00:19.56Z" }, ] +[[package]] +name = "rasterio" +version = "1.4.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12'", +] +dependencies = [ + { name = "affine", marker = "python_full_version < '3.12'" }, + { name = "attrs", marker = "python_full_version < '3.12'" }, + { name = "certifi", marker = "python_full_version < '3.12'" }, + { name = "click", marker = "python_full_version < '3.12'" }, + { name = "click-plugins", marker = "python_full_version < '3.12'" }, + { name = "cligj", marker = "python_full_version < '3.12'" }, + { name = "numpy", marker = "python_full_version < '3.12'" }, + { name = "pyparsing", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ec/fa/fce8dc9f09e5bc6520b6fc1b4ecfa510af9ca06eb42ad7bdff9c9b8989d0/rasterio-1.4.4.tar.gz", hash = "sha256:c95424e2c7f009b8f7df1095d645c52895cd332c0c2e1b4c2e073ea28b930320", size = 445004, upload-time = "2025-12-12T18:01:08.971Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/0d/d3859e49ab94464de2623fec82c6798d8d7c8bea2473cd2696fc5e09f717/rasterio-1.4.4-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b8eea428b5f0c78a963f6003a19b60777df83a0aba8c28231d65431e32ac160e", size = 21144125, upload-time = "2025-12-12T17:58:59.511Z" }, + { url = "https://files.pythonhosted.org/packages/aa/3c/97ba4b146309cdc0e36f289b02ac69465b026a21afc828e4e4e1dc39466a/rasterio-1.4.4-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:1cc0ea5aa0d22f5f349aa221674481de689b7b3a99607ce6bb58a29e5be54d17", size = 25746406, upload-time = "2025-12-12T17:59:02.902Z" }, + { url = "https://files.pythonhosted.org/packages/ce/33/75f81bd837ac2336b24456fdb249597a4b9af2a212b7151f64d09022be36/rasterio-1.4.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7eb25b23666b29dadfc49a59206cead62c99190584b61771bba0e95f7da06801", size = 34587242, upload-time = "2025-12-12T17:59:05.848Z" }, + { url = "https://files.pythonhosted.org/packages/f9/77/3869a426f6e752dde13f3868cdf16253ca0214f92107db79c1583c9aa07b/rasterio-1.4.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e24b7b8c2df801dde2a1dffb44c58902bd76b5cab740dc11de4ff9963992a71a", size = 35881871, upload-time = "2025-12-12T17:59:09.779Z" }, + { url = "https://files.pythonhosted.org/packages/66/d0/3818859ddbd3750d0ef5a6580a3272e81764286d943c689dd41e49b8b786/rasterio-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:0718630f607be2f5742d8e4b34b434746fd788a192d77eefc9bb924399fea802", size = 25716477, upload-time = "2025-12-12T17:59:13.519Z" }, + { url = "https://files.pythonhosted.org/packages/4b/02/039eb4970c93aaef4c9eb1ee159abad18e6e7f932c2eed575c95f78d94f6/rasterio-1.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:0308ff4762ae9eb40a991f12d758626b59af4376b13675480391dd7295d17bbf", size = 24075993, upload-time = "2025-12-12T17:59:16.407Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fc/63d89ddfcb4643730553683ee322566b9b15fe56d026e4c21c4f4f5d9d26/rasterio-1.4.4-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f3c4f0cbd188f893011f2a0a6dc2852b3892799b3a0d79eddf92f2b115ec7ed7", size = 21120715, upload-time = "2025-12-12T17:59:19.35Z" }, + { url = "https://files.pythonhosted.org/packages/43/70/2c003f76a23dbb078fdee35c8e2ec490d2ad8982f4dc956ba08b56027b87/rasterio-1.4.4-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:6fce26090b9f509eab337228420145947c491a13628965410f25bc3e6e05cf75", size = 25732944, upload-time = "2025-12-12T17:59:22.533Z" }, + { url = "https://files.pythonhosted.org/packages/f6/cc/4a8e92362c0ff496dd1007c3dcba66e9ededf1a45eca8ad1db302b071c49/rasterio-1.4.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c1c722da390dc264aeccdc0dc200ca37923875d910ca4cd5bec0fec351bb818e", size = 34295209, upload-time = "2025-12-12T17:59:26.035Z" }, + { url = "https://files.pythonhosted.org/packages/e6/6d/717d2dec47fbefad33ca0d27bd5f0d543b1d1bc9fcab5ef82a13adaaf38d/rasterio-1.4.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:98b6dfb8282b2a54b9d75c3dc8d2520a69bbc66916c7d43de8e0bbf6e0240ca1", size = 35661866, upload-time = "2025-12-12T17:59:29.928Z" }, + { url = "https://files.pythonhosted.org/packages/ed/60/ae3351fba2726ec0976974ce2eb030c159edd3363b8771e832b8db571c24/rasterio-1.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:9513f4c7a6d93b45098f8dff2421fa9516604e3bfbf35aa144484a88d36a321f", size = 25682853, upload-time = "2025-12-12T17:59:35.869Z" }, + { url = "https://files.pythonhosted.org/packages/38/ee/35387296bbacfc5cbbb4273228b1b959793d3ce38b0402a07f11a248420b/rasterio-1.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:60b49a482e0f12f12ce9d2cc3090add02f89f3d422e85f2cffaa9207adb83c04", size = 24043249, upload-time = "2025-12-12T17:59:39.915Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fe/e3e37041c49956f4f4cbe473c3fe290aaba96ed20e9c07da304e0cad2015/rasterio-1.4.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:df26c96aa81ffbd0b33189680859211eadf9950123c21579f84de73bb0f91d81", size = 21107336, upload-time = "2025-12-12T17:59:43.585Z" }, + { url = "https://files.pythonhosted.org/packages/f3/02/c217fdcc8e80a4b7d1b1bc4529d78f98452816e9add53ff8742049a77ae7/rasterio-1.4.4-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:b3af0ecc922a80f3755516629f7948e37bade9077b5f5c12a3869a5e7f01619b", size = 25719929, upload-time = "2025-12-12T17:59:47.64Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d0/7f177f37bc9595d809dabb0073abd0c42358469f6b10875192b46331c652/rasterio-1.4.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7ce3b0f9a22e95a27790087908753973644d7c3877d495ec9bd6e04a25233ca4", size = 34198845, upload-time = "2025-12-12T17:59:52.405Z" }, + { url = "https://files.pythonhosted.org/packages/7b/84/66c0d9cca2a09074ec2ce6fffa87709ca51b0d197ae742d835e841bac660/rasterio-1.4.4-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c072450caa96428b1218b030500bb908fd6f09bc013a88969ff81a124b6a112a", size = 35576074, upload-time = "2025-12-12T17:59:56.392Z" }, + { url = "https://files.pythonhosted.org/packages/32/68/f7df5478458ace2fa50be43e9fab1a39957a0e71afaa3e6147ec289e0fc8/rasterio-1.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:16ee92ef10c0ba89f45f9c2b40fca9f971f357385f04ee9b716fb09cbd9ce20c", size = 25680573, upload-time = "2025-12-12T18:00:00.45Z" }, + { url = "https://files.pythonhosted.org/packages/34/e5/1bdaccb658430dfd391ad4a63d206546f36639d7e4130bf31f125c6525b4/rasterio-1.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:65c10afe64b5e488185aaff0b659e08eda22c89285b54a3e433b80e6c6621770", size = 24040367, upload-time = "2025-12-12T18:00:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/32/76/54643a7d1d650fd7f1acea9093c298603e4c01bba6f90be2254310b48507/rasterio-1.4.4-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:18c2c1130e789dc2771d0aa5ec4b56d5b8a0097c648ccb94882d5ff3ab55c928", size = 21247203, upload-time = "2025-12-12T18:00:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/76/ef/434b4849ccd6a3e03a0b1ac37c963c1771564945745613d15c5d96ce768d/rasterio-1.4.4-cp313-cp313t-macosx_15_0_x86_64.whl", hash = "sha256:2d1654b7ffa6f3dde42c5fd27159ae45148c11e352de26f12fe7313a3236aeed", size = 25822050, upload-time = "2025-12-12T18:00:11.081Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fa/fe9a478aa0cde246da58baeb0df3248c7ca174e4d9c9b27e81b504e40a76/rasterio-1.4.4-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c4022cbddb659856e120603b12233cec8913ae760fff220657ce888c3c6b9f9d", size = 34833783, upload-time = "2025-12-12T18:00:14.525Z" }, + { url = "https://files.pythonhosted.org/packages/04/cd/ed4716590dbcd4b8ae633417d758564e510bee4d6aaac5050a0f6d5179c5/rasterio-1.4.4-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:96b88880551a07b7a3b50439483cefbd9af91a09e19ff2b736815994e5671314", size = 35738114, upload-time = "2025-12-12T18:00:17.96Z" }, + { url = "https://files.pythonhosted.org/packages/7e/29/da7050d11ba1d041e0333ac14768e6e9ca1aa2b9fa8416f317d2650ed276/rasterio-1.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:def75d486d0ab8f306f918a913c425ed57159495518c54efe8e18d5164d37d90", size = 25896835, upload-time = "2025-12-12T18:00:21.411Z" }, + { url = "https://files.pythonhosted.org/packages/88/80/304dbe5434c4aa8dfaf90480c16d770161796a6a61fa88e72e8a402153df/rasterio-1.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:770b7e86f6c565e6f9cf30f6fa4479a5a2bab4e10ff44fe7acfd518ca4a71d1b", size = 24128074, upload-time = "2025-12-12T18:00:24.653Z" }, + { url = "https://files.pythonhosted.org/packages/03/01/d5a3dc51cd5fef62b76ecc77d33c1ca20de305fed7e16c71bcdf4858e466/rasterio-1.4.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:019693f14a83ae9225cb57c16e466901d0e6284962dcf13a9f4bb1175b979011", size = 21120237, upload-time = "2025-12-12T18:00:27.723Z" }, + { url = "https://files.pythonhosted.org/packages/50/da/db18362602b17327c0e00c9e9c0847c1c4ac657c1a289169ca06a26faccb/rasterio-1.4.4-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:87d7c3e97e3b40c9041d1602e2dcb4fc2d88abe6c645fccb4939dec297a91cf8", size = 25720506, upload-time = "2025-12-12T18:00:30.592Z" }, + { url = "https://files.pythonhosted.org/packages/5a/8f/a15d66c9c05bffb176c9707ef1f2bfcf9c0b835272937c80ac7207a20b5c/rasterio-1.4.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a2401e4c43a31c7382154d4042b60a63b9bca5886802983c5c9362cdc5b09548", size = 34153931, upload-time = "2025-12-12T18:00:33.852Z" }, + { url = "https://files.pythonhosted.org/packages/05/2d/cd778286b910db7a3f0bc1743ca362173f1fbb7365137e4982ca857b6d26/rasterio-1.4.4-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:6c4287d8934d953f7870b8e2a1df1096fbf47eba39ad0f777a31ea500f4e5010", size = 35421139, upload-time = "2025-12-12T18:00:37.482Z" }, + { url = "https://files.pythonhosted.org/packages/70/97/13a2e33aede8d7a42178c696a6a93868d1f9560f73de05033a1675f0806a/rasterio-1.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:c3ba1871549221140661227dd4fa1f9a472ded4a6d2f2c2e367b0648bb15b99d", size = 26419132, upload-time = "2025-12-12T18:00:40.871Z" }, + { url = "https://files.pythonhosted.org/packages/27/d8/2dcfcb362d6a2fd07c14cfb803a345a7926d4d9fb6243e196df105671e97/rasterio-1.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:7c9d7dc824cb8d222808be153643cd4e65ea3e1f66019ada1ccd630221edfe30", size = 24800998, upload-time = "2025-12-12T18:00:45.332Z" }, + { url = "https://files.pythonhosted.org/packages/13/f8/16e9b648e7f16cadb41df7c0116dbab26b4a2ba02c85cbe3f744065bdf56/rasterio-1.4.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:98e17bded830a59992d9f8f8d9f227ce1c4be0694930afcc4360358f5cb1a5db", size = 21247046, upload-time = "2025-12-12T18:00:49.429Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ea/f3dc3a25d7591821d488f5c5eb89f6abcd1f5c8e2ef4bd2792f965cbc9c8/rasterio-1.4.4-cp314-cp314t-macosx_15_0_x86_64.whl", hash = "sha256:56134ca203f952855e60774b06672033cf65057eb9810fcc5c1a75f1921053a3", size = 25821677, upload-time = "2025-12-12T18:00:52.458Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d3/1e038350218e852f904c8dc4ab751aa023a2e82e68998767b7b42e33832c/rasterio-1.4.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:52edde65515b33fe4314c8a44a9ee2fc00b550deed6d56e1a8d085d42bbca3e6", size = 34829572, upload-time = "2025-12-12T18:00:56.294Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ce/28abf7a5f5d9cb014c2e14cc396bebe953b3deefbf604d49f4322e73fa35/rasterio-1.4.4-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d61d3f2c171c64050bd75e54a5d964ff7f165b3f5d2b92c9ee09b9716aa1b8bf", size = 35735171, upload-time = "2025-12-12T18:00:59.531Z" }, + { url = "https://files.pythonhosted.org/packages/54/91/1ce35cfda2d56dacd6395faf20a5290268bd9009c53393ac42b5f9bb2c4c/rasterio-1.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:40137fe512c0d6e96c0167a0ae4e56d82c488f244163c45494b7392e51c844de", size = 26700712, upload-time = "2025-12-12T18:01:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/3b/33/4d13f48a8f01d782ffc1eece20821586518f3f515dca7cf152bca9fd22d4/rasterio-1.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:29ec3a794454b5bb255c9c0374cc380030a8a1e295c81eee7feb036802d2a9e3", size = 24875933, upload-time = "2025-12-12T18:01:06.134Z" }, +] + +[[package]] +name = "rasterio" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", +] +dependencies = [ + { name = "affine", marker = "python_full_version >= '3.12'" }, + { name = "attrs", marker = "python_full_version >= '3.12'" }, + { name = "certifi", marker = "python_full_version >= '3.12'" }, + { name = "click", marker = "python_full_version >= '3.12'" }, + { name = "cligj", marker = "python_full_version >= '3.12'" }, + { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "pyparsing", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/88/edb4b66b6cb2c13f123af5a3896bf70c0cbe73ab3cd4243cb4eb0212a0f6/rasterio-1.5.0.tar.gz", hash = "sha256:1e0ea56b02eea4989b36edf8e58a5a3ef40e1b7edcb04def2603accd5ab3ee7b", size = 452184, upload-time = "2026-01-05T16:06:47.169Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/de/ba1cd11d7d1182bfb26e758bf07016d04e5442f4f5fea35b0d7279b72399/rasterio-1.5.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:420656074897a460f5ef46f657b3061d2e004f9d99e613914b0671643e69d92c", size = 22787192, upload-time = "2026-01-05T16:05:19.779Z" }, + { url = "https://files.pythonhosted.org/packages/e6/42/efaeb6dc531dbcd02fec01c791a853bb5a139a5126ecec579ac0f735eeb9/rasterio-1.5.0-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:c5c3597a783857e760550e8f26365d928b0377ac5ffc3e12ba447ac65ca5406d", size = 24412221, upload-time = "2026-01-05T16:05:22.526Z" }, + { url = "https://files.pythonhosted.org/packages/a2/14/89645988424c40cbcb8334f94305ffe094dd28d85c643341d9690704c9f0/rasterio-1.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e14d07a09833b6df6024ce7a57aee1e1977b3aec682e30b1e58ce773462f2382", size = 36128020, upload-time = "2026-01-05T16:05:25.556Z" }, + { url = "https://files.pythonhosted.org/packages/85/23/5a52319a98451ff910f42e5f7f4804bfb39f9327933a89daab685d1ce2dd/rasterio-1.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:26dbcffcf0d01fc121cbb92186bc1cb78e16efe62b17be45ad7494446b325cf8", size = 37634010, upload-time = "2026-01-05T16:05:28.673Z" }, + { url = "https://files.pythonhosted.org/packages/57/d6/fe8826f813c98b046d8d4c3bc83053c89c71f367f89257d211fe5dd0b0ba/rasterio-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:ac8d04eee66ca8060763ead607800e5611d857dd005905d920365e24a16ba20a", size = 30142328, upload-time = "2026-01-05T16:05:31.357Z" }, + { url = "https://files.pythonhosted.org/packages/af/62/6397379271d5628ed65ef781bf2d3a8f56094a86e6d8479c6ca506a1b960/rasterio-1.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:31f1edc45c781ebd087e60cc00a4fc37028dd3fe25cff4098e4139fc9d0565be", size = 28500710, upload-time = "2026-01-05T16:05:33.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/87/42865a77cebf2e524d27b6afc71db48984799ecd1dbe6a213d4713f42f5f/rasterio-1.5.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e7b25b0a19975ccd511e507e6de45b0a2d8fb6802abe49bb726cf48588e34833", size = 22776107, upload-time = "2026-01-05T16:05:36.967Z" }, + { url = "https://files.pythonhosted.org/packages/6a/53/e81683fbbfdf04e019e68b042d9cff8524b0571aa80e4f4d81c373c31a49/rasterio-1.5.0-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:1162c18eaece9f6d2aa1c2ff6b373b99651d93f113f24120a991eaebf28aa4f4", size = 24401477, upload-time = "2026-01-05T16:05:39.702Z" }, + { url = "https://files.pythonhosted.org/packages/bc/3c/6aa6e0690b18eea02a61739cb362a47c5df66138f0a02cc69e1181b964e5/rasterio-1.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:8eb87fd6f843eea109f3df9bef83f741b053b716b0465932276e2c0577dfb929", size = 36018214, upload-time = "2026-01-05T16:05:42.741Z" }, + { url = "https://files.pythonhosted.org/packages/48/4a/1af9aa9810fb30668568f2c4dd3eec2412c8e9762b69201d971c509b295e/rasterio-1.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:08a7580cbb9b3bd320bdf827e10c9b2424d0df066d8eef6f2feb37e154ce0c17", size = 37544972, upload-time = "2026-01-05T16:05:45.815Z" }, + { url = "https://files.pythonhosted.org/packages/01/62/bfe3408743c9837919ff232474a09ece9eaa88d4ee8c040711fa3dff6dad/rasterio-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:d7d6729c0739b5ec48c33686668a30e27f5bdb361093f180ee7818ff19665547", size = 30140141, upload-time = "2026-01-05T16:05:48.751Z" }, + { url = "https://files.pythonhosted.org/packages/63/ca/e90e19a6d065a718cc3d468a12b9f015289ad17017656dea8c76f7318d1f/rasterio-1.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:8af7c368c22f0a99d1259ccc5a5cd96c432c2bde6f132c1ac78508cd7445a745", size = 28498556, upload-time = "2026-01-05T16:05:51.334Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ba/e37462d8c33bbbd6c152a0390ec6911a3d9614ded3d2bc6f6a48e147e833/rasterio-1.5.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:b4ccfcc8ed9400e4f14efdf2005533fcf72048748b727f85ff89b9291ecdf98a", size = 22920107, upload-time = "2026-01-05T16:05:53.773Z" }, + { url = "https://files.pythonhosted.org/packages/66/dc/7bfa9cf96ac39b451b2f94dfc584c223ec584c52c148df2e4bab60c3341b/rasterio-1.5.0-cp313-cp313t-macosx_15_0_x86_64.whl", hash = "sha256:2f57c36ca4d3c896f7024226bd71eeb5cd10c8183c2a94508534d78cc05ff9e7", size = 24508993, upload-time = "2026-01-05T16:05:57.062Z" }, + { url = "https://files.pythonhosted.org/packages/e5/55/7293743f3b69de4b726c67b8dc9da01fc194070b6becc51add4ca8a20a27/rasterio-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cc1395475e4bb7032cd81dda4d5558061c4c7d5a50b1b5e146bdf9716d0b9353", size = 36565784, upload-time = "2026-01-05T16:06:00.019Z" }, + { url = "https://files.pythonhosted.org/packages/cf/ef/5354c47de16c6e289728c3a3d6961ffcf7a9ad6313aef7e8db5d6a40c46e/rasterio-1.5.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:592a485e2057b1aaeab4f843c9897628e60e3ff45e2509325c3e1479116599cb", size = 37686456, upload-time = "2026-01-05T16:06:02.772Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fc/fe1f034b1acd1900d9fbd616826d001a3d5811f1d0c97c785f88f525853e/rasterio-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:0c739e70a72fb080f039ee1570c5d02b974dde32ded1a3216e1f13fe38ac4844", size = 30355842, upload-time = "2026-01-05T16:06:06.359Z" }, + { url = "https://files.pythonhosted.org/packages/e0/cb/4dee9697891c9c6474b240d00e27688e03ecd882d3c83cc97eb25c2266ff/rasterio-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:a3539a2f401a7b4b2e94ff2db334878c0e15a2d1c9fe90bb0879c52f89367ae5", size = 28589538, upload-time = "2026-01-05T16:06:09.662Z" }, + { url = "https://files.pythonhosted.org/packages/77/9f/f84dfa54110c1c82f9f4fd929465d12519569b6f5d015273aa0957013b2e/rasterio-1.5.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:597be8df418d5ba7b6a927b6b9febfcb42b192882448a8d5b2e2e75a1296631f", size = 22788832, upload-time = "2026-01-05T16:06:12.247Z" }, + { url = "https://files.pythonhosted.org/packages/20/f1/de55255c918b17afd7292f793a3500c4aea7e9530b2b3f5b3a57836c7d49/rasterio-1.5.0-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:dd292030d39d685c0b35eddef233e7f1cb8b43052578a3ec97a2da57799693be", size = 24405917, upload-time = "2026-01-05T16:06:14.603Z" }, + { url = "https://files.pythonhosted.org/packages/a9/57/054087a9d5011ad5dfa799277ba8814e41775e1967d37a59ab7b8e2f1876/rasterio-1.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:62c3f97a3c72643c74f2d0f310621a09c35c0c412229c327ae6bcc1ee4b9c3bc", size = 35987536, upload-time = "2026-01-05T16:06:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/c9/72/5fbe5f67ae75d7e89ffb718c500d5fecbaa84f6ba354db306de689faf961/rasterio-1.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:19577f0f0c5f1158af47b57f73356961cbd1782a5f6ae6f3adf6f2650f4eb369", size = 37408048, upload-time = "2026-01-05T16:06:20.82Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3e/0c4ef19980204bdcbc8f9e084056adebc97916ff4edcc718750ef34e5bf9/rasterio-1.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:015c1ab6e5453312c5e29692752e7ad73568fe4d13567cbd448d7893128cbd2d", size = 30949590, upload-time = "2026-01-05T16:06:23.425Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d8/2e6b81505408926c00e629d7d3d73fd0454213201bd9907450e0fe82f3dd/rasterio-1.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:ff677c0a9d3ba667c067227ef2b76872488b37ff29b061bc3e576fad9baa3286", size = 29337287, upload-time = "2026-01-05T16:06:26.599Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/7b6e6afb28d4e3f69f2229f990ed87dfdc21a3e15ca63b96b2fd9ba17d89/rasterio-1.5.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:508251b9c746d8d008771a30c2160ff321bfc3b41f6a1aa8e8ef1dd4a00d97ba", size = 22926149, upload-time = "2026-01-05T16:06:29.617Z" }, + { url = "https://files.pythonhosted.org/packages/24/30/19345d8bc7d2b96c1172594026b9009702e9ab9f0baf07079d3612aaadae/rasterio-1.5.0-cp314-cp314t-macosx_15_0_x86_64.whl", hash = "sha256:742841ed48bc70f6ef517b8fa3521f231780bf408fde0aa6d73770337a36374e", size = 24516040, upload-time = "2026-01-05T16:06:32.964Z" }, + { url = "https://files.pythonhosted.org/packages/9e/43/dc7a4518fa78904bc41952cbf346c3c2a88a20e61b479154058392914c0b/rasterio-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c9a9eee49ce9410c2f352b34c370bb3a96bb518b6a7f97b3a72ee4c835fd4b5c", size = 36589519, upload-time = "2026-01-05T16:06:35.922Z" }, + { url = "https://files.pythonhosted.org/packages/8f/f2/8f706083c6c163054d12c7ed6d5ac4e4ed02252b761288d74e6158871b34/rasterio-1.5.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b9fd87a0b63ab5c6267dfb0bc96f54fdf49d000651b9ee85ed37798141cff046", size = 37714599, upload-time = "2026-01-05T16:06:38.818Z" }, + { url = "https://files.pythonhosted.org/packages/a6/d5/bbca726d5fea5864f7e4bcf3ee893095369e93ad51120495e8c40e2aa1a0/rasterio-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f459db8953ba30ca04fcef2b5e1260eeeff0eae8158bd9c3d6adbe56289765cc", size = 31233931, upload-time = "2026-01-05T16:06:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d1/8b017856e63ccaff3cbd0e82490dbb01363a42f3a462a41b1d8a391e1443/rasterio-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f4b9c2c3b5f10469eb9588f105086e68f0279e62cc9095c4edd245e3f9b88c8a", size = 29418321, upload-time = "2026-01-05T16:06:44.758Z" }, +] + [[package]] name = "requests" version = "2.32.5" @@ -2256,6 +2629,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/9e/e11e9b008d6bd9f8febb6475eb1964e3ee1437ef70d2885b707a95e686cb/shap-0.48.0-cp313-cp313-win_amd64.whl", hash = "sha256:d1f92323452c5e3ad4338621c7c848cda3059e88ea85338a443e671204c20384", size = 545108, upload-time = "2025-06-12T13:05:24.665Z" }, ] +[[package]] +name = "shapely" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/8d/1ff672dea9ec6a7b5d422eb6d095ed886e2e523733329f75fdcb14ee1149/shapely-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:91121757b0a36c9aac3427a651a7e6567110a4a67c97edf04f8d55d4765f6618", size = 1820038, upload-time = "2025-09-24T13:50:15.628Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ce/28fab8c772ce5db23a0d86bf0adaee0c4c79d5ad1db766055fa3dab442e2/shapely-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:16a9c722ba774cf50b5d4541242b4cce05aafd44a015290c82ba8a16931ff63d", size = 1626039, upload-time = "2025-09-24T13:50:16.881Z" }, + { url = "https://files.pythonhosted.org/packages/70/8b/868b7e3f4982f5006e9395c1e12343c66a8155c0374fdc07c0e6a1ab547d/shapely-2.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cc4f7397459b12c0b196c9efe1f9d7e92463cbba142632b4cc6d8bbbbd3e2b09", size = 3001519, upload-time = "2025-09-24T13:50:18.606Z" }, + { url = "https://files.pythonhosted.org/packages/13/02/58b0b8d9c17c93ab6340edd8b7308c0c5a5b81f94ce65705819b7416dba5/shapely-2.1.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:136ab87b17e733e22f0961504d05e77e7be8c9b5a8184f685b4a91a84efe3c26", size = 3110842, upload-time = "2025-09-24T13:50:21.77Z" }, + { url = "https://files.pythonhosted.org/packages/af/61/8e389c97994d5f331dcffb25e2fa761aeedfb52b3ad9bcdd7b8671f4810a/shapely-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:16c5d0fc45d3aa0a69074979f4f1928ca2734fb2e0dde8af9611e134e46774e7", size = 4021316, upload-time = "2025-09-24T13:50:23.626Z" }, + { url = "https://files.pythonhosted.org/packages/d3/d4/9b2a9fe6039f9e42ccf2cb3e84f219fd8364b0c3b8e7bbc857b5fbe9c14c/shapely-2.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6ddc759f72b5b2b0f54a7e7cde44acef680a55019eb52ac63a7af2cf17cb9cd2", size = 4178586, upload-time = "2025-09-24T13:50:25.443Z" }, + { url = "https://files.pythonhosted.org/packages/16/f6/9840f6963ed4decf76b08fd6d7fed14f8779fb7a62cb45c5617fa8ac6eab/shapely-2.1.2-cp311-cp311-win32.whl", hash = "sha256:2fa78b49485391224755a856ed3b3bd91c8455f6121fee0db0e71cefb07d0ef6", size = 1543961, upload-time = "2025-09-24T13:50:26.968Z" }, + { url = "https://files.pythonhosted.org/packages/38/1e/3f8ea46353c2a33c1669eb7327f9665103aa3a8dfe7f2e4ef714c210b2c2/shapely-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:c64d5c97b2f47e3cd9b712eaced3b061f2b71234b3fc263e0fcf7d889c6559dc", size = 1722856, upload-time = "2025-09-24T13:50:28.497Z" }, + { url = "https://files.pythonhosted.org/packages/24/c0/f3b6453cf2dfa99adc0ba6675f9aaff9e526d2224cbd7ff9c1a879238693/shapely-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe2533caae6a91a543dec62e8360fe86ffcdc42a7c55f9dfd0128a977a896b94", size = 1833550, upload-time = "2025-09-24T13:50:30.019Z" }, + { url = "https://files.pythonhosted.org/packages/86/07/59dee0bc4b913b7ab59ab1086225baca5b8f19865e6101db9ebb7243e132/shapely-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ba4d1333cc0bc94381d6d4308d2e4e008e0bd128bdcff5573199742ee3634359", size = 1643556, upload-time = "2025-09-24T13:50:32.291Z" }, + { url = "https://files.pythonhosted.org/packages/26/29/a5397e75b435b9895cd53e165083faed5d12fd9626eadec15a83a2411f0f/shapely-2.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0bd308103340030feef6c111d3eb98d50dc13feea33affc8a6f9fa549e9458a3", size = 2988308, upload-time = "2025-09-24T13:50:33.862Z" }, + { url = "https://files.pythonhosted.org/packages/b9/37/e781683abac55dde9771e086b790e554811a71ed0b2b8a1e789b7430dd44/shapely-2.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e7d4d7ad262a48bb44277ca12c7c78cb1b0f56b32c10734ec9a1d30c0b0c54b", size = 3099844, upload-time = "2025-09-24T13:50:35.459Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f3/9876b64d4a5a321b9dc482c92bb6f061f2fa42131cba643c699f39317cb9/shapely-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9eddfe513096a71896441a7c37db72da0687b34752c4e193577a145c71736fc", size = 3988842, upload-time = "2025-09-24T13:50:37.478Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a0/704c7292f7014c7e74ec84eddb7b109e1fbae74a16deae9c1504b1d15565/shapely-2.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:980c777c612514c0cf99bc8a9de6d286f5e186dcaf9091252fcd444e5638193d", size = 4152714, upload-time = "2025-09-24T13:50:39.9Z" }, + { url = "https://files.pythonhosted.org/packages/53/46/319c9dc788884ad0785242543cdffac0e6530e4d0deb6c4862bc4143dcf3/shapely-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9111274b88e4d7b54a95218e243282709b330ef52b7b86bc6aaf4f805306f454", size = 1542745, upload-time = "2025-09-24T13:50:41.414Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bf/cb6c1c505cb31e818e900b9312d514f381fbfa5c4363edfce0fcc4f8c1a4/shapely-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:743044b4cfb34f9a67205cee9279feaf60ba7d02e69febc2afc609047cb49179", size = 1722861, upload-time = "2025-09-24T13:50:43.35Z" }, + { url = "https://files.pythonhosted.org/packages/c3/90/98ef257c23c46425dc4d1d31005ad7c8d649fe423a38b917db02c30f1f5a/shapely-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b510dda1a3672d6879beb319bc7c5fd302c6c354584690973c838f46ec3e0fa8", size = 1832644, upload-time = "2025-09-24T13:50:44.886Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ab/0bee5a830d209adcd3a01f2d4b70e587cdd9fd7380d5198c064091005af8/shapely-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8cff473e81017594d20ec55d86b54bc635544897e13a7cfc12e36909c5309a2a", size = 1642887, upload-time = "2025-09-24T13:50:46.735Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5e/7d7f54ba960c13302584c73704d8c4d15404a51024631adb60b126a4ae88/shapely-2.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe7b77dc63d707c09726b7908f575fc04ff1d1ad0f3fb92aec212396bc6cfe5e", size = 2970931, upload-time = "2025-09-24T13:50:48.374Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a2/83fc37e2a58090e3d2ff79175a95493c664bcd0b653dd75cb9134645a4e5/shapely-2.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ed1a5bbfb386ee8332713bf7508bc24e32d24b74fc9a7b9f8529a55db9f4ee6", size = 3082855, upload-time = "2025-09-24T13:50:50.037Z" }, + { url = "https://files.pythonhosted.org/packages/44/2b/578faf235a5b09f16b5f02833c53822294d7f21b242f8e2d0cf03fb64321/shapely-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a84e0582858d841d54355246ddfcbd1fce3179f185da7470f41ce39d001ee1af", size = 3979960, upload-time = "2025-09-24T13:50:51.74Z" }, + { url = "https://files.pythonhosted.org/packages/4d/04/167f096386120f692cc4ca02f75a17b961858997a95e67a3cb6a7bbd6b53/shapely-2.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc3487447a43d42adcdf52d7ac73804f2312cbfa5d433a7d2c506dcab0033dfd", size = 4142851, upload-time = "2025-09-24T13:50:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/48/74/fb402c5a6235d1c65a97348b48cdedb75fb19eca2b1d66d04969fc1c6091/shapely-2.1.2-cp313-cp313-win32.whl", hash = "sha256:9c3a3c648aedc9f99c09263b39f2d8252f199cb3ac154fadc173283d7d111350", size = 1541890, upload-time = "2025-09-24T13:50:55.337Z" }, + { url = "https://files.pythonhosted.org/packages/41/47/3647fe7ad990af60ad98b889657a976042c9988c2807cf322a9d6685f462/shapely-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:ca2591bff6645c216695bdf1614fca9c82ea1144d4a7591a466fef64f28f0715", size = 1722151, upload-time = "2025-09-24T13:50:57.153Z" }, + { url = "https://files.pythonhosted.org/packages/3c/49/63953754faa51ffe7d8189bfbe9ca34def29f8c0e34c67cbe2a2795f269d/shapely-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2d93d23bdd2ed9dc157b46bc2f19b7da143ca8714464249bef6771c679d5ff40", size = 1834130, upload-time = "2025-09-24T13:50:58.49Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ee/dce001c1984052970ff60eb4727164892fb2d08052c575042a47f5a9e88f/shapely-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01d0d304b25634d60bd7cf291828119ab55a3bab87dc4af1e44b07fb225f188b", size = 1642802, upload-time = "2025-09-24T13:50:59.871Z" }, + { url = "https://files.pythonhosted.org/packages/da/e7/fc4e9a19929522877fa602f705706b96e78376afb7fad09cad5b9af1553c/shapely-2.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8d8382dd120d64b03698b7298b89611a6ea6f55ada9d39942838b79c9bc89801", size = 3018460, upload-time = "2025-09-24T13:51:02.08Z" }, + { url = "https://files.pythonhosted.org/packages/a1/18/7519a25db21847b525696883ddc8e6a0ecaa36159ea88e0fef11466384d0/shapely-2.1.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:19efa3611eef966e776183e338b2d7ea43569ae99ab34f8d17c2c054d3205cc0", size = 3095223, upload-time = "2025-09-24T13:51:04.472Z" }, + { url = "https://files.pythonhosted.org/packages/48/de/b59a620b1f3a129c3fecc2737104a0a7e04e79335bd3b0a1f1609744cf17/shapely-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:346ec0c1a0fcd32f57f00e4134d1200e14bf3f5ae12af87ba83ca275c502498c", size = 4030760, upload-time = "2025-09-24T13:51:06.455Z" }, + { url = "https://files.pythonhosted.org/packages/96/b3/c6655ee7232b417562bae192ae0d3ceaadb1cc0ffc2088a2ddf415456cc2/shapely-2.1.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6305993a35989391bd3476ee538a5c9a845861462327efe00dd11a5c8c709a99", size = 4170078, upload-time = "2025-09-24T13:51:08.584Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8e/605c76808d73503c9333af8f6cbe7e1354d2d238bda5f88eea36bfe0f42a/shapely-2.1.2-cp313-cp313t-win32.whl", hash = "sha256:c8876673449f3401f278c86eb33224c5764582f72b653a415d0e6672fde887bf", size = 1559178, upload-time = "2025-09-24T13:51:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/36/f7/d317eb232352a1f1444d11002d477e54514a4a6045536d49d0c59783c0da/shapely-2.1.2-cp313-cp313t-win_amd64.whl", hash = "sha256:4a44bc62a10d84c11a7a3d7c1c4fe857f7477c3506e24c9062da0db0ae0c449c", size = 1739756, upload-time = "2025-09-24T13:51:12.105Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c4/3ce4c2d9b6aabd27d26ec988f08cb877ba9e6e96086eff81bfea93e688c7/shapely-2.1.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:9a522f460d28e2bf4e12396240a5fc1518788b2fcd73535166d748399ef0c223", size = 1831290, upload-time = "2025-09-24T13:51:13.56Z" }, + { url = "https://files.pythonhosted.org/packages/17/b9/f6ab8918fc15429f79cb04afa9f9913546212d7fb5e5196132a2af46676b/shapely-2.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ff629e00818033b8d71139565527ced7d776c269a49bd78c9df84e8f852190c", size = 1641463, upload-time = "2025-09-24T13:51:14.972Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/91d59ae525ca641e7ac5551c04c9503aee6f29b92b392f31790fcb1a4358/shapely-2.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f67b34271dedc3c653eba4e3d7111aa421d5be9b4c4c7d38d30907f796cb30df", size = 2970145, upload-time = "2025-09-24T13:51:16.961Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cb/4948be52ee1da6927831ab59e10d4c29baa2a714f599f1f0d1bc747f5777/shapely-2.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21952dc00df38a2c28375659b07a3979d22641aeb104751e769c3ee825aadecf", size = 3073806, upload-time = "2025-09-24T13:51:18.712Z" }, + { url = "https://files.pythonhosted.org/packages/03/83/f768a54af775eb41ef2e7bec8a0a0dbe7d2431c3e78c0a8bdba7ab17e446/shapely-2.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1f2f33f486777456586948e333a56ae21f35ae273be99255a191f5c1fa302eb4", size = 3980803, upload-time = "2025-09-24T13:51:20.37Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/559c7c195807c91c79d38a1f6901384a2878a76fbdf3f1048893a9b7534d/shapely-2.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cf831a13e0d5a7eb519e96f58ec26e049b1fad411fc6fc23b162a7ce04d9cffc", size = 4133301, upload-time = "2025-09-24T13:51:21.887Z" }, + { url = "https://files.pythonhosted.org/packages/80/cd/60d5ae203241c53ef3abd2ef27c6800e21afd6c94e39db5315ea0cbafb4a/shapely-2.1.2-cp314-cp314-win32.whl", hash = "sha256:61edcd8d0d17dd99075d320a1dd39c0cb9616f7572f10ef91b4b5b00c4aeb566", size = 1583247, upload-time = "2025-09-24T13:51:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/74/d4/135684f342e909330e50d31d441ace06bf83c7dc0777e11043f99167b123/shapely-2.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:a444e7afccdb0999e203b976adb37ea633725333e5b119ad40b1ca291ecf311c", size = 1773019, upload-time = "2025-09-24T13:51:24.873Z" }, + { url = "https://files.pythonhosted.org/packages/a3/05/a44f3f9f695fa3ada22786dc9da33c933da1cbc4bfe876fe3a100bafe263/shapely-2.1.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:5ebe3f84c6112ad3d4632b1fd2290665aa75d4cef5f6c5d77c4c95b324527c6a", size = 1834137, upload-time = "2025-09-24T13:51:26.665Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/4d57db45bf314573427b0a70dfca15d912d108e6023f623947fa69f39b72/shapely-2.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5860eb9f00a1d49ebb14e881f5caf6c2cf472c7fd38bd7f253bbd34f934eb076", size = 1642884, upload-time = "2025-09-24T13:51:28.029Z" }, + { url = "https://files.pythonhosted.org/packages/5a/27/4e29c0a55d6d14ad7422bf86995d7ff3f54af0eba59617eb95caf84b9680/shapely-2.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b705c99c76695702656327b819c9660768ec33f5ce01fa32b2af62b56ba400a1", size = 3018320, upload-time = "2025-09-24T13:51:29.903Z" }, + { url = "https://files.pythonhosted.org/packages/9f/bb/992e6a3c463f4d29d4cd6ab8963b75b1b1040199edbd72beada4af46bde5/shapely-2.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a1fd0ea855b2cf7c9cddaf25543e914dd75af9de08785f20ca3085f2c9ca60b0", size = 3094931, upload-time = "2025-09-24T13:51:32.699Z" }, + { url = "https://files.pythonhosted.org/packages/9c/16/82e65e21070e473f0ed6451224ed9fa0be85033d17e0c6e7213a12f59d12/shapely-2.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:df90e2db118c3671a0754f38e36802db75fe0920d211a27481daf50a711fdf26", size = 4030406, upload-time = "2025-09-24T13:51:34.189Z" }, + { url = "https://files.pythonhosted.org/packages/7c/75/c24ed871c576d7e2b64b04b1fe3d075157f6eb54e59670d3f5ffb36e25c7/shapely-2.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:361b6d45030b4ac64ddd0a26046906c8202eb60d0f9f53085f5179f1d23021a0", size = 4169511, upload-time = "2025-09-24T13:51:36.297Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f7/b3d1d6d18ebf55236eec1c681ce5e665742aab3c0b7b232720a7d43df7b6/shapely-2.1.2-cp314-cp314t-win32.whl", hash = "sha256:b54df60f1fbdecc8ebc2c5b11870461a6417b3d617f555e5033f1505d36e5735", size = 1602607, upload-time = "2025-09-24T13:51:37.757Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/f09272a71976dfc138129b8faf435d064a811ae2f708cb147dccdf7aacdb/shapely-2.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:0036ac886e0923417932c2e6369b6c52e38e0ff5d9120b90eef5cd9a5fc5cae9", size = 1796682, upload-time = "2025-09-24T13:51:39.233Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -2313,7 +2745,7 @@ wheels = [ [[package]] name = "st-forecast" -version = "0.1.8" +version = "0.1.9" source = { editable = "." } dependencies = [ { name = "click" }, @@ -2338,6 +2770,10 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "bump-my-version" }, + { name = "cartopy" }, + { name = "contextily" }, + { name = "geopandas" }, + { name = "matplotlib-scalebar" }, { name = "pytest" }, { name = "ruff" }, ] @@ -2366,6 +2802,10 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "bump-my-version", specifier = ">=1.2.7" }, + { name = "cartopy", specifier = ">=0.25.0" }, + { name = "contextily", specifier = ">=1.7.0" }, + { name = "geopandas", specifier = ">=1.1.3" }, + { name = "matplotlib-scalebar", specifier = ">=0.9.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "ruff", specifier = ">=0.13.2" }, ] @@ -2627,6 +3067,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/a7/6eeb32e705d510a672f74135f538ad27f87f3d600845bfd3834ea3a77c7e/xarray-2025.9.1-py3-none-any.whl", hash = "sha256:3e9708db0d7915c784ed6c227d81b398dca4957afe68d119481f8a448fc88c44", size = 1364411, upload-time = "2025-09-30T05:28:51.294Z" }, ] +[[package]] +name = "xyzservices" +version = "2026.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/08/3cb9f67a8d48021aca2a02292cc26eecd71d949ae70ad66420a8730cc302/xyzservices-2026.3.0.tar.gz", hash = "sha256:d226866a5d8e9fef337034d8da37a8298f0a1d9d1489b4018e69579eb321fea4", size = 1135736, upload-time = "2026-03-30T14:42:25.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a9/d23012099dc88ec69a29c6407b41d89681cb674c2043cd5b467c7e299c08/xyzservices-2026.3.0-py3-none-any.whl", hash = "sha256:503183d4b322bfebc3c50cdd21192aa3e81e36c5efbf9133d54ae82143e0576b", size = 94101, upload-time = "2026-03-30T14:42:24.608Z" }, +] + [[package]] name = "yarl" version = "1.20.1" From e7316dfcd7a80697ea9b90b77f0f5f566c918773 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Wed, 15 Apr 2026 11:10:30 +0200 Subject: [PATCH 03/20] fixed columns --- st_forecast/operational/sapphire_predictor.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/st_forecast/operational/sapphire_predictor.py b/st_forecast/operational/sapphire_predictor.py index 1bd5f47..f793e10 100644 --- a/st_forecast/operational/sapphire_predictor.py +++ b/st_forecast/operational/sapphire_predictor.py @@ -313,7 +313,8 @@ def predict( pred_df = create_prediction_df( prediction, self.quantiles, date_col=self.config.date_col ) - pred_df[self.config.basin_id_col] = code + basin_id_col = self.config.basin_id_col + pred_df[basin_id_col] = code return inverse_scale_predictions( pred_df, self.scalers, self.config, self.static_df ) @@ -343,7 +344,7 @@ def hindcast( ------- pd.DataFrame Hindcast predictions with columns: - date, code, forecast_step, forecast_issue_date, Q5..Q95 + date, Q5..Q95, forecast_date """ if n > self.config.forecast_horizon: logging.warning( @@ -354,7 +355,6 @@ def hindcast( n = self.config.forecast_horizon code = int(identifier) date_col = self.config.date_col - basin_id_col = self.config.basin_id_col # Prepare DataFrames (same as predict — no mode distinction) df_discharge, df_covariates, df_covariates_past = self._prepare_data( @@ -384,19 +384,19 @@ def hindcast( predict_kwargs=predict_kwargs, ) - # Process results + # Process results — match BaseDartsDLPredictor output schema: + # date, Q columns, forecast_date all_dfs = [] for ts in backtests: df = create_prediction_df(ts, self.quantiles, date_col=date_col) - df["forecast_step"] = range(1, len(df) + 1) + min_date = df[date_col].min() + df["forecast_date"] = min_date - pd.DateOffset(days=1) all_dfs.append(df) result = pd.concat(all_dfs, ignore_index=True) - result[basin_id_col] = code - result["forecast_issue_date"] = result[date_col] - pd.to_timedelta( - result["forecast_step"], unit="D" - ) + basin_id_col = self.config.basin_id_col + result[basin_id_col] = code return inverse_scale_predictions( result, self.scalers, self.config, self.static_df ) From 035da564316c129b27ea9bb038efc4cb0657d283 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Wed, 15 Apr 2026 15:28:43 +0200 Subject: [PATCH 04/20] removing custom lstm --- .gitignore | 3 +- configs/examples/rnn_lstm_caravan.json | 78 +++ configs/examples/tft_sapphire.json | 73 +++ configs/examples/tide_caravan.json | 78 +++ configs/examples/tsmixer_sapphire.json | 78 +++ configs/tft_kgz.json | 1 + configs/tide_kgz.json | 1 + docs/configuration.md | 4 +- docs/models.md | 12 +- docs/user-guide.md | 2 +- pyproject.toml | 1 - scripts/explain_tft.py | 7 +- st_forecast/custom_models/ExoLSTM.py | 20 +- st_forecast/custom_models/ExoMamba.py | 422 ------------- st_forecast/custom_models/lstm/__init__.py | 3 - st_forecast/custom_models/lstm/model.py | 99 --- st_forecast/custom_models/lstm/module.py | 207 ------ .../custom_models/lstm_enc_dec/__init__.py | 3 - .../custom_models/lstm_enc_dec/model.py | 103 --- .../custom_models/lstm_enc_dec/module.py | 248 -------- .../custom_models/lstm_enc_mlp/__init__.py | 3 - .../custom_models/lstm_enc_mlp/model.py | 109 ---- .../custom_models/lstm_enc_mlp/module.py | 267 -------- .../custom_models/mamba_enc_dec/__init__.py | 3 - .../custom_models/mamba_enc_dec/model.py | 103 --- .../custom_models/mamba_enc_dec/module.py | 167 ----- .../unified_temporal/__init__.py | 4 - .../unified_temporal/backends/__init__.py | 15 - .../unified_temporal/backends/attention.py | 97 --- .../unified_temporal/backends/base.py | 22 - .../unified_temporal/backends/lstm.py | 44 -- .../unified_temporal/backends/mamba.py | 52 -- .../unified_temporal/backends/tcn.py | 108 ---- .../unified_temporal/feature_utils.py | 125 ---- .../custom_models/unified_temporal/model.py | 217 ------- .../custom_models/unified_temporal/module.py | 297 --------- st_forecast/model_helper.py | 205 +----- st_forecast/pipeline/artifacts.py | 8 +- st_forecast/pipeline/inference.py | 16 - st_forecast/pipeline/training.py | 17 +- st_forecast/train_model.py | 8 - tests/custom_models/test_lstm_enc_dec.py | 594 ------------------ tests/test_inference.py | 18 - tests/test_model_initialization.py | 48 +- tests/test_model_type_normalization.py | 2 - tests/test_temporal_backends.py | 130 ---- uv.lock | 14 - 47 files changed, 388 insertions(+), 3748 deletions(-) create mode 100644 configs/examples/rnn_lstm_caravan.json create mode 100644 configs/examples/tft_sapphire.json create mode 100644 configs/examples/tide_caravan.json create mode 100644 configs/examples/tsmixer_sapphire.json delete mode 100644 st_forecast/custom_models/ExoMamba.py delete mode 100644 st_forecast/custom_models/lstm/__init__.py delete mode 100644 st_forecast/custom_models/lstm/model.py delete mode 100644 st_forecast/custom_models/lstm/module.py delete mode 100644 st_forecast/custom_models/lstm_enc_dec/__init__.py delete mode 100644 st_forecast/custom_models/lstm_enc_dec/model.py delete mode 100644 st_forecast/custom_models/lstm_enc_dec/module.py delete mode 100644 st_forecast/custom_models/lstm_enc_mlp/__init__.py delete mode 100644 st_forecast/custom_models/lstm_enc_mlp/model.py delete mode 100644 st_forecast/custom_models/lstm_enc_mlp/module.py delete mode 100644 st_forecast/custom_models/mamba_enc_dec/__init__.py delete mode 100644 st_forecast/custom_models/mamba_enc_dec/model.py delete mode 100644 st_forecast/custom_models/mamba_enc_dec/module.py delete mode 100644 st_forecast/custom_models/unified_temporal/__init__.py delete mode 100644 st_forecast/custom_models/unified_temporal/backends/__init__.py delete mode 100644 st_forecast/custom_models/unified_temporal/backends/attention.py delete mode 100644 st_forecast/custom_models/unified_temporal/backends/base.py delete mode 100644 st_forecast/custom_models/unified_temporal/backends/lstm.py delete mode 100644 st_forecast/custom_models/unified_temporal/backends/mamba.py delete mode 100644 st_forecast/custom_models/unified_temporal/backends/tcn.py delete mode 100644 st_forecast/custom_models/unified_temporal/feature_utils.py delete mode 100644 st_forecast/custom_models/unified_temporal/model.py delete mode 100644 st_forecast/custom_models/unified_temporal/module.py delete mode 100644 tests/custom_models/test_lstm_enc_dec.py delete mode 100644 tests/test_temporal_backends.py diff --git a/.gitignore b/.gitignore index a8ac20b..7bd8863 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,8 @@ __pycache__/ # Project specific /runs /lightning_logs -/configs +/configs/* +!/configs/examples/ /.serena /st_forecast/data_utils/__pycache__ /st_forecast/data_utils/__pycache__ diff --git a/configs/examples/rnn_lstm_caravan.json b/configs/examples/rnn_lstm_caravan.json new file mode 100644 index 0000000..0b7fb04 --- /dev/null +++ b/configs/examples/rnn_lstm_caravan.json @@ -0,0 +1,78 @@ +{ + "region": "lamah_ce", + "data_source": "caravan", + + "caravan": { + "base_path": "/path/to/Caravan/Caravan-nc", + "region": "lamah", + "n_basins": 10, + "variable_mapping": { + "streamflow": "discharge", + "total_precipitation_sum": "P", + "temperature_2m_mean": "T", + "potential_evaporation_sum_FAO_PENMAN_MONTEITH": "PET", + "snow_depth_water_equivalent_mean": "SWE" + } + }, + + "model_type": "RNN", + "model_name": "RNN_LSTM_LAMAH", + "encoder": "month", + "input_chunk_length": 30, + "forecast_horizon": 10, + + "exog_features": ["P", "T", "PET", "SWE"], + "past_features": [], + "future_features": ["P", "T", "PET"], + "static_features": [], + + "moving_windows": [3, 5, 10], + "shift_moving_avg": false, + + "scale_discharge_bool": true, + "global_scaling": true, + "transform_log": false, + "transform_mm": false, + + "discharge_scaler_type": "standard", + "forcing_scaler_type": "standard", + "static_scaler_type": "minmax", + + "hidden_size": 128, + "rnn_model": "LSTM", + "n_rnn_layers": 1, + "training_length": 50, + "dropout": 0.2, + "n_epochs": 30, + "weight_decay": 1e-5, + "lr": 0.0005, + "batch_size": 1028, + + "factor": 0.5, + "patience": 3, + "early_stopping_patience": 5, + + "scheduler_type": "cosine", + "scheduler_t_max": 30, + "scheduler_eta_min": 5e-5, + + "early_stopping_enabled": true, + "early_stopping_min_delta": 0.001, + + "likelihood_type": "quantile", + "gradient_clip_val": 0.1, + "accelerator": "auto", + "checkpoint_strategy": "last", + "mc_dropout": true, + + "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95], + "num_samples": 100, + + "work_dir": "/path/to/checkpoints/", + + "train_years": [1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005], + "val_years": [2005, 2006, 2007, 2008], + "test_years": [2009, 2010, 2011, 2012, 2013, 2014, 2015], + + "use_only_natural_rivers": false +} diff --git a/configs/examples/tft_sapphire.json b/configs/examples/tft_sapphire.json new file mode 100644 index 0000000..2eb8226 --- /dev/null +++ b/configs/examples/tft_sapphire.json @@ -0,0 +1,73 @@ +{ + "region": "KGZ", + + "path_to_operational_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/control_member_forcing", + "path_to_hindcast_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/hindcast_forcing", + "HRU_forcing": "00003", + "path_rivers": "../../../data/xy_data_forecast_tools/intermediate_data/runoff_day.csv", + "path_static": "../../../data/xy_data_forecast_tools/config/models_and_scalers/static_features/ML_basin_attributes_v2.csv", + "path_to_sla": "../../../data/sla/fsc_sla_timeseries_gapfilled.csv", + "path_to_nir": "../../../data/sla/meanNIR_TS_allBasins.csv", + "path_to_swe": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/SWE", + "path_to_hs": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/HS", + "path_to_rof": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/RoF", + + "model_type": "TFT", + "model_name": "TFT_KGZ", + "encoder": "month", + "input_chunk_length": 30, + "forecast_horizon": 10, + + "exog_features": ["P", "T", "PET", "daylight_hours"], + "past_features": ["moving_avr_dis_3", "moving_avr_dis_5", "moving_avr_dis_10"], + "future_features": ["P", "T", "PET", "daylight_hours"], + + "moving_windows": [3, 5, 10], + "shift_moving_avg": false, + + "scale_discharge_bool": true, + "global_scaling": true, + "transform_log": false, + "transform_mm": true, + + "discharge_scaler_type": "standard", + "forcing_scaler_type": "standard", + "static_scaler_type": "standard", + + "n_epochs": 30, + "lr": 0.0005, + "weight_decay": 1e-5, + "hidden_size": 32, + "dropout": 0.3, + "batch_size": 1028, + "num_attention_heads": 4, + + "factor": 0.5, + "patience": 3, + "early_stopping_patience": 10, + + "scheduler_type": "reduce_on_plateau", + + "early_stopping_enabled": true, + "early_stopping_min_delta": 0.001, + + "likelihood_type": "quantile", + "gradient_clip_val": 0.1, + "optimizer": "Adam", + "accelerator": "auto", + "checkpoint_strategy": "last", + "mc_dropout": true, + + "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95], + "num_samples": 200, + + "work_dir": "/path/to/checkpoints/", + + "train_years": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016], + "val_years": [2017, 2018, 2019, 2020, 2021], + "test_years": [2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025], + + "codes_to_remove": [], + "unnatural_rivers": [], + "use_only_natural_rivers": false +} diff --git a/configs/examples/tide_caravan.json b/configs/examples/tide_caravan.json new file mode 100644 index 0000000..912207c --- /dev/null +++ b/configs/examples/tide_caravan.json @@ -0,0 +1,78 @@ +{ + "region": "lamah_ce", + "data_source": "caravan", + + "caravan": { + "base_path": "/path/to/Caravan/Caravan-nc", + "region": "lamah", + "n_basins": 10, + "variable_mapping": { + "streamflow": "discharge", + "total_precipitation_sum": "P", + "temperature_2m_mean": "T", + "potential_evaporation_sum_FAO_PENMAN_MONTEITH": "PET", + "snow_depth_water_equivalent_mean": "SWE" + } + }, + + "model_type": "TiDE", + "model_name": "TiDE_LAMAH", + "encoder": "month", + "input_chunk_length": 30, + "forecast_horizon": 10, + + "exog_features": ["P", "T", "PET", "SWE"], + "past_features": ["moving_avr_dis_3", "moving_avr_dis_5", "moving_avr_dis_10"], + "future_features": ["P", "T", "PET", "SWE"], + "static_features": [], + + "moving_windows": [3, 5, 10], + "shift_moving_avg": false, + + "scale_discharge_bool": true, + "global_scaling": true, + "transform_log": false, + "transform_mm": false, + + "discharge_scaler_type": "standard", + "forcing_scaler_type": "standard", + "static_scaler_type": "minmax", + + "n_epochs": 30, + "weight_decay": 4.35e-05, + "lr": 0.0005, + "hidden_size": 64, + "dropout": 0.5, + "num_layers": 1, + "temporal_decoder_hidden_size": 32, + "decoder_output_dim": 16, + "batch_size": 1028, + + "factor": 0.5, + "patience": 3, + "early_stopping_patience": 10, + + "scheduler_type": "cosine", + "scheduler_t_max": 20, + "scheduler_eta_min": 5e-5, + + "early_stopping_enabled": true, + "early_stopping_min_delta": 0.001, + + "likelihood_type": "quantile", + "gradient_clip_val": 0.1, + "accelerator": "auto", + "checkpoint_strategy": "last", + "mc_dropout": true, + + "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95], + "num_samples": 200, + + "work_dir": "/path/to/checkpoints/", + + "train_years": [1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005], + "val_years": [2005, 2006, 2007, 2008], + "test_years": [2009, 2010, 2011, 2012, 2013, 2014, 2015], + + "use_only_natural_rivers": false +} diff --git a/configs/examples/tsmixer_sapphire.json b/configs/examples/tsmixer_sapphire.json new file mode 100644 index 0000000..f7a78f7 --- /dev/null +++ b/configs/examples/tsmixer_sapphire.json @@ -0,0 +1,78 @@ +{ + "region": "KGZ", + + "path_to_operational_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/control_member_forcing", + "path_to_hindcast_forcing": "../../../data/xy_data_forecast_tools/intermediate_data/hindcast_forcing", + "HRU_forcing": "00003", + "path_rivers": "../../../data/xy_data_forecast_tools/intermediate_data/runoff_day.csv", + "path_static": "../../../data/xy_data_forecast_tools/config/models_and_scalers/static_features/ML_basin_attributes_v2.csv", + "path_to_sla": "../../../data/sla/fsc_sla_timeseries_gapfilled.csv", + "path_to_nir": "../../../data/sla/meanNIR_TS_allBasins.csv", + "path_to_swe": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/SWE", + "path_to_hs": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/HS", + "path_to_rof": "../../../data/xy_data_forecast_tools/intermediate_data/snow_data/RoF", + + "model_type": "TSMixer", + "model_name": "TSMixer_KGZ", + "encoder": "month", + "input_chunk_length": 30, + "forecast_horizon": 10, + + "exog_features": ["P", "T", "PET", "ROF", "SWE", "daylight_hours"], + "past_features": ["moving_avr_dis_3", "moving_avr_dis_5", "moving_avr_dis_10"], + "future_features": ["P", "T", "PET", "daylight_hours"], + + "moving_windows": [3, 5, 10], + "shift_moving_avg": false, + + "scale_discharge_bool": true, + "global_scaling": true, + "transform_log": false, + "transform_mm": true, + + "discharge_scaler_type": "standard", + "forcing_scaler_type": "standard", + "static_scaler_type": "standard", + + "n_epochs": 40, + "weight_decay": 1e-5, + "lr": 0.0005, + "hidden_size": 64, + "dropout": 0.2, + "num_blocks": 2, + "ff_size": 64, + "activation": "ReLU", + "normalize_before": false, + "norm_type": "LayerNorm", + "batch_size": 2048, + + "factor": 0.5, + "patience": 3, + "early_stopping_patience": 10, + + "scheduler_type": "cosine", + "scheduler_t_max": 20, + "scheduler_eta_min": 5e-5, + + "early_stopping_enabled": true, + "early_stopping_min_delta": 0.001, + + "likelihood_type": "quantile", + "gradient_clip_val": 0.1, + "accelerator": "auto", + "checkpoint_strategy": "last", + "mc_dropout": true, + + "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95], + "num_samples": 200, + + "work_dir": "/path/to/checkpoints/", + + "train_years": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016], + "val_years": [2017, 2018, 2019, 2020, 2021], + "test_years": [2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025], + + "codes_to_remove": [], + "unnatural_rivers": [], + "use_only_natural_rivers": false +} diff --git a/configs/tft_kgz.json b/configs/tft_kgz.json index 9c2f6f9..bc85366 100644 --- a/configs/tft_kgz.json +++ b/configs/tft_kgz.json @@ -14,6 +14,7 @@ "model_type": "TFT", "model_name": "TFT_KGZ", + "encoder": "month", "input_chunk_length": 30, "forecast_horizon": 10, diff --git a/configs/tide_kgz.json b/configs/tide_kgz.json index 81fbbfe..4cd8170 100644 --- a/configs/tide_kgz.json +++ b/configs/tide_kgz.json @@ -14,6 +14,7 @@ "model_type": "TiDE", "model_name": "TiDE_KGZ", + "encoder": "month", "input_chunk_length": 30, "forecast_horizon": 10, diff --git a/docs/configuration.md b/docs/configuration.md index 8d3dd27..32bc8cc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,7 +99,7 @@ Note: `features` is a computed property (`exog_features + past_features`) and sh | Parameter | Type | Description | |-----------|------|-------------| -| `model_type` | str | `"TiDE"`, `"TFT"`, `"TSMixer"`, `"FANMixer"`, `"ExoMamba"`, `"ExoLSTM"`, `"UnifiedTemporal"`, `"LSTMEncDec"`, `"MambaEncDec"` (case-insensitive) | +| `model_type` | str | `"TiDE"`, `"TFT"`, `"TSMixer"`, `"FANMixer"`, `"ExoLSTM"`, `"RNN"` (case-insensitive) | | `model_name` | str | Identifier for saving checkpoints | ## Hyperparameters @@ -120,6 +120,7 @@ Note: `features` is a computed property (`exog_features + past_features`) and sh | `gradient_clip_val` | float | 0.1 | Gradient clipping | | `n_epochs` | int | 40 | Training epochs | | `optimizer` | str | `"adam"` | `"adam"`, `"adamw"`, `"sgd"` | +| `encoder` | str | `"month"` | Cyclic temporal encoder: `"week"`, `"month"`, or `"none"` (disables encoding) | | `extra_hyperparams` | dict | {} | Model-specific hyperparameters not explicitly defined above (passed through to model) | ## Early Stopping @@ -231,6 +232,7 @@ Based on `configs/tide_kgz.json`: "gradient_clip_val": 0.1, "accelerator": "auto", "checkpoint_strategy": "last", + "encoder": "month", "work_dir": "/path/to/checkpoints/", "train_years": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016], diff --git a/docs/models.md b/docs/models.md index 80cc032..94bd246 100644 --- a/docs/models.md +++ b/docs/models.md @@ -4,18 +4,11 @@ | Model | Type | Key Features | |-------|------|--------------| -| LSTMEncDec | LSTM Encoder-Decoder | Skip connections, learned init from static | -| MambaEncDec | Mamba Encoder-Decoder | State-space model, efficient long sequences | -| UnifiedTemporal | Flexible backbone | Supports LSTM, Mamba, TCN, Attention backends | | FANMixer | MLP-Mixer | Fourier analysis, time/feature mixing | | TFT | Darts wrapper | Temporal Fusion Transformer via Darts | | TiDE | Darts wrapper | Time-series Dense Encoder via Darts | | TSMixer | Darts wrapper | TimeSeries Mixer via Darts | -## Deprecated Models - -- **ExoLSTM, ExoMamba**: Replaced by LSTMEncDec and MambaEncDec - ## Common Features - **Quantile Likelihood**: All models support quantile regression for probabilistic forecasting @@ -25,10 +18,7 @@ ## Model Selection Guidance -- **LSTMEncDec**: Recommended for most use cases - stable, fast, skip connections help gradient flow -- **UnifiedTemporal**: When you want to experiment with different backends (lstm, mamba, tcn, attention) - **TFT**: When interpretability (attention weights) is important -- **MambaEncDec**: For very long sequences where LSTM struggles ## Configuration @@ -36,7 +26,7 @@ All models are configured via `model_type` in config.json: ```json { - "model_type": "LSTMEncDec", + "model_type": "ExoLSTM", "hidden_size": 64, "n_layers": 1, "dropout": 0.3, diff --git a/docs/user-guide.md b/docs/user-guide.md index 5ed20b3..6f09abe 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -442,7 +442,7 @@ from pathlib import Path config = RunConfig.from_json(Path("config.json")) print(config.region) # e.g., "KGZ" -print(config.model_type) # e.g., "TiDE", "LSTMEncDec" +print(config.model_type) # e.g., "TiDE", "ExoLSTM" print(config.input_chunk_length) # e.g., 30 print(config.forecast_horizon) # e.g., 10 print(config.past_features) # list of feature names diff --git a/pyproject.toml b/pyproject.toml index 259cee6..e9987b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ dependencies = [ "darts==0.43.0", "joblib>=1.5.2", "lightning==2.5.1", - "mambapy==1.2.0", "matplotlib>=3.10.6", "numpy>=2.3.3", "optuna>=4.5.0", diff --git a/scripts/explain_tft.py b/scripts/explain_tft.py index 12ca219..a7ab6fd 100644 --- a/scripts/explain_tft.py +++ b/scripts/explain_tft.py @@ -158,8 +158,11 @@ def _clean_col(name: str) -> str: # Strip darts suffixes name = re.sub(r"_(pastcov|futcov|statcov|target)$", "", name) # Darts cyclic encodings → clean names - name = re.sub(r"darts_enc_fc_cyc_week_cos", "Week Cos", name) - name = re.sub(r"darts_enc_fc_cyc_week_sin", "Week Sin", name) + name = re.sub( + r"darts_enc_fc_cyc_(\w+)_(cos|sin)", + lambda m: f"{m.group(1).title()} {m.group(2).title()}", + name, + ) # Moving averages: moving_avr_dis_N → MA{N} name = re.sub(r"moving_avr_dis_(\d+)", r"MA\1", name) return name diff --git a/st_forecast/custom_models/ExoLSTM.py b/st_forecast/custom_models/ExoLSTM.py index ca95387..19de2e3 100644 --- a/st_forecast/custom_models/ExoLSTM.py +++ b/st_forecast/custom_models/ExoLSTM.py @@ -1,14 +1,4 @@ -""" -Exogenous-only LSTM (ExoLSTM) ------- - -.. deprecated:: - ExoLSTM is deprecated. Use UnifiedTemporalModel with - temporal_model="lstm" and use_past_covariates=False instead. -""" - -import warnings -from typing import Optional +"""Exogenous-only LSTM model that uses only forcing/exogenous variables.""" import torch import torch.nn as nn @@ -114,7 +104,7 @@ def __init__( @io_processor def forward( - self, x_in: tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]] + self, x_in: tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None] ) -> torch.Tensor: """ExoLSTM model forward pass. Parameters @@ -293,12 +283,6 @@ def __init__( >>> model.fit(target, past_covariates=past_cov, future_covariates=future_cov) >>> pred = model.predict(6, future_covariates=future_cov[100:106]) """ - warnings.warn( - "ExoLSTMModel is deprecated. Use UnifiedTemporalModel with " - "temporal_model='lstm' and use_past_covariates=False instead.", - DeprecationWarning, - stacklevel=2, - ) super().__init__(**self._extract_torch_model_params(**self.model_params)) # extract pytorch lightning module kwargs diff --git a/st_forecast/custom_models/ExoMamba.py b/st_forecast/custom_models/ExoMamba.py deleted file mode 100644 index d84fda9..0000000 --- a/st_forecast/custom_models/ExoMamba.py +++ /dev/null @@ -1,422 +0,0 @@ -""" -Exogenous-only Mamba (ExoMamba) ------- - -.. deprecated:: - ExoMamba is deprecated. Use UnifiedTemporalModel with - temporal_model="mamba" and use_past_covariates=False instead. -""" - -import warnings -from typing import Optional - -import torch -import torch.nn as nn - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.pl_forecasting_module import ( - PLForecastingModule, - io_processor, -) -from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel -from darts.utils.torch import MonteCarloDropout - -try: - from mambapy.mamba import Mamba, MambaConfig -except ImportError: - raise ImportError( - "Mamba is not installed. Please install it using `pip install mambapy`." - ) - -MixedCovariatesTrainTensorType = tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor -] - -logger = get_logger(__name__) - - -class _ExoMambaModule(PLForecastingModule): - def __init__( - self, - input_dim: int, - output_dim: int, - dynamic_cov_dim: int, # Now using the same dimension for both past and future covariates - static_cov_dim: int, - nr_params: int, - hidden_size: int, - n_mamba_layers: int, - d_state: int, - expand_factor: int, - dropout: float, - **kwargs, - ): - """Pytorch module implementing the ExoMamba architecture. - - Parameters - ---------- - input_dim - The number of input components (target + optional past covariates + optional future covariates). - output_dim - Number of output components in the target. - dynamic_cov_dim - Number of dynamic covariates (same for both past and future). - static_cov_dim - Number of static covariates. - nr_params - The number of parameters of the likelihood (or 1 if no likelihood is used). - hidden_size - The size of the hidden layers in the model. - n_mamba_layers - Number of Mamba layers in the model. - d_state - State dimension for Mamba blocks. - expand_factor - Expansion factor for inner dimension in Mamba blocks. - dropout - Dropout probability - **kwargs - All parameters required for :class:`darts.models.forecasting.pl_forecasting_module.PLForecastingModule` - base class. - - Inputs - ------ - x - Tuple of Tensors `(x_past, x_future, x_static)` where `x_past` is the input/past chunk and - `x_future`is the output/future chunk. Input dimensions are `(batch_size, time_steps, components)` - Outputs - ------- - y - Tensor of shape `(batch_size, output_chunk_length, output_dim, nr_params)` - """ - - super().__init__(**kwargs) - - self.input_dim = input_dim - self.output_dim = output_dim - self.dynamic_cov_dim = dynamic_cov_dim - self.static_cov_dim = static_cov_dim - self.nr_params = nr_params - self.hidden_size = hidden_size - self.dropout = dropout - - # Mamba configuration - self.n_mamba_layers = n_mamba_layers - self.d_state = d_state - self.expand_factor = expand_factor - - # Combined embedding layer for dynamic+static features - combined_dim = dynamic_cov_dim - if static_cov_dim > 0: - combined_dim += static_cov_dim - - self.combined_embedding = nn.Linear(combined_dim, hidden_size) - - # Dropout and activation - self.dropout_layer = MonteCarloDropout(dropout) - self.activation = nn.SiLU() - - # Mamba model - self.mamba_config = MambaConfig( - d_model=hidden_size, - n_layers=n_mamba_layers, - d_state=d_state, - expand_factor=expand_factor, - ) - self.mamba = Mamba(self.mamba_config) - - # Output projection - self.output_proj = nn.Linear(hidden_size, output_dim * self.nr_params) - - @io_processor - def forward( - self, x_in: tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]] - ) -> torch.Tensor: - """ExoMamba model forward pass. - Parameters - ---------- - x_in - comes as tuple `(x_past, x_future, x_static)` where `x_past` is the input/past chunk and `x_future` - is the output/future chunk. Input dimensions are `(batch_size, time_steps, components)` - Returns - ------- - torch.Tensor - The output Tensor of shape `(batch_size, output_chunk_length, output_dim, nr_params)` - """ - - # x has shape (batch_size, input_chunk_length, input_dim) - # x_future_covariates has shape (batch_size, output_chunk_length, future_cov_dim) - # x_static_covariates has shape (batch_size, static_cov_dim) - x, x_future_covariates, x_static_covariates = x_in - - batch_size = x.shape[0] - - # Extract past covariates from x (excluding autoregressive features) - # The autoregressive features are the first self.output_dim components - x_past_covariates = x[ - :, :, self.output_dim : self.output_dim + self.dynamic_cov_dim - ] - - # Verify that future covariates are provided and have the same dimension - if x_future_covariates is None: - raise_log( - ValueError( - "ExoMamba requires future covariates with the same dimension as past covariates." - ), - logger=logger, - ) - - if x_future_covariates.shape[2] != self.dynamic_cov_dim: - raise_log( - ValueError( - f"Future covariates dimension ({x_future_covariates.shape[2]}) must match past covariates dimension ({self.dynamic_cov_dim})." - ), - logger=logger, - ) - - # Process static covariates if available - if self.static_cov_dim > 0 and x_static_covariates is not None: - # Ensure x_static_covariates has the right shape - if len(x_static_covariates.shape) > 2: - # Flatten any extra dimensions - x_static_covariates = x_static_covariates.view(batch_size, -1) - - # Expand static to match sequence length for past covariates - past_seq_length = x_past_covariates.shape[1] - static_past = x_static_covariates.unsqueeze(1).expand( - batch_size, past_seq_length, -1 - ) - - # Expand static to match sequence length for future covariates - future_seq_length = x_future_covariates.shape[1] - static_future = x_static_covariates.unsqueeze(1).expand( - batch_size, future_seq_length, -1 - ) - - # Concatenate dynamic and static features along feature dimension - past_combined = torch.cat([x_past_covariates, static_past], dim=2) - future_combined = torch.cat([x_future_covariates, static_future], dim=2) - else: - past_combined = x_past_covariates - future_combined = x_future_covariates - - # Embed past and future combined features - past_embedded = self.combined_embedding(past_combined) - past_embedded = self.activation(past_embedded) - past_embedded = self.dropout_layer(past_embedded) - - future_embedded = self.combined_embedding(future_combined) - future_embedded = self.activation(future_embedded) - future_embedded = self.dropout_layer(future_embedded) - - # Concatenate past and future along time dimension - sequence_embedded = torch.cat([past_embedded, future_embedded], dim=1) - - # Process through Mamba - mamba_output = self.mamba(sequence_embedded) - - # Take the output_chunk_length last time steps - # This ensures we're only forecasting for the future time steps - last_steps = mamba_output[:, -self.output_chunk_length :] - - # Project to output dimensions - output = self.output_proj(last_steps) - - # Reshape to (batch_size, output_chunk_length, output_dim, nr_params) - output = output.view( - batch_size, self.output_chunk_length, self.output_dim, self.nr_params - ) - - # Final NaN check on output - if torch.isnan(output).any(): - logger.warning("NaN values detected in final output. Replacing with zeros.") - print("Output shape:", output.shape) - - return output - - -class ExoMambaModel(MixedCovariatesTorchModel): - def __init__( - self, - input_chunk_length: int, - output_chunk_length: int, - output_chunk_shift: int = 0, - hidden_size: int = 64, - n_mamba_layers: int = 2, - d_state: int = 16, - expand_factor: int = 2, - dropout: float = 0.1, - use_static_covariates: bool = True, - **kwargs, - ): - """An implementation of the ExoMamba model, which uses only exogenous features for forecasting. - - This model requires both past and future covariates with the same dimensionality, as they - are concatenated along the time dimension to form a single sequence. The model ignores - autoregressive components of the target series and relies solely on: - - - past covariates (known for `input_chunk_length` points before prediction time) - - future covariates (known for `output_chunk_length` points after prediction time) - - optional static covariates - - The model is suitable for cases where autoregressive information is not available or - when the emphasis is on using external factors for prediction. - - Parameters - ---------- - input_chunk_length - Number of time steps in the past to take as a model input (per chunk). This applies to - past covariates only, as the model ignores the target series historical values. - output_chunk_length - Number of time steps predicted at once (per chunk) by the internal model. Also, the number of future values - from future covariates to use as a model input. - output_chunk_shift - Optionally, the number of steps to shift the start of the output chunk into the future (relative to the - input chunk end). This will create a gap between the input and output chunks. - hidden_size - The size of the hidden layers in the model. - n_mamba_layers - Number of Mamba layers to use in the model. - d_state - State dimension for Mamba blocks. - expand_factor - Expansion factor for inner dimension in Mamba blocks. - dropout - The dropout probability to be used in the model. This is compatible with Monte Carlo dropout - at inference time for model uncertainty estimation (enabled with ``mc_dropout=True`` at - prediction time). - use_static_covariates - Whether the model should use static covariates. - default_nan_handling - Strategy for handling NaN values in inputs and during computation: - - "zero": Replace NaN values with zeros (default) - - "mean": Replace NaN values with the mean of the tensor - - "error": Raise an error when NaN values are detected - **kwargs - Optional arguments to initialize the pytorch_lightning.Module, pytorch_lightning.Trainer, and - Darts' :class:`TorchForecastingModel`. - - Examples - -------- - >>> from darts.datasets import WeatherDataset - >>> from darts.models import ExoMambaModel - >>> series = WeatherDataset().load() - >>> # predicting atmospheric pressure using only covariates - >>> target = series['p (mbar)'][:100] - >>> # past observed covariates (must have same dimension as future covariates) - >>> past_cov = series[['rain (mm)', 'T (degC)']][:100] - >>> # future covariates (must have same dimension as past covariates) - >>> future_cov = series[['rain (mm)', 'T (degC)']][:106] - >>> model = ExoMambaModel( - >>> input_chunk_length=6, - >>> output_chunk_length=6, - >>> n_epochs=20 - >>> ) - >>> model.fit(target, past_covariates=past_cov, future_covariates=future_cov) - >>> pred = model.predict(6, future_covariates=future_cov[100:106]) - """ - warnings.warn( - "ExoMambaModel is deprecated. Use UnifiedTemporalModel with " - "temporal_model='mamba' and use_past_covariates=False instead.", - DeprecationWarning, - stacklevel=2, - ) - super().__init__(**self._extract_torch_model_params(**self.model_params)) - - # extract pytorch lightning module kwargs - self.pl_module_params = self._extract_pl_module_params(**self.model_params) - - self.hidden_size = hidden_size - self.n_mamba_layers = n_mamba_layers - self.d_state = d_state - self.expand_factor = expand_factor - self.dropout = dropout - self._considers_static_covariates = use_static_covariates - - def _create_model( - self, train_sample: MixedCovariatesTrainTensorType - ) -> torch.nn.Module: - ( - past_target, - past_covariates, - historic_future_covariates, - future_covariates, - static_covariates, - future_target, - ) = train_sample - - # Target dimension - output_dim = future_target.shape[1] - - # Verify that we have past and future covariates - if past_covariates is None or future_covariates is None: - raise_log( - ValueError("ExoMamba requires both past and future covariates."), - logger=logger, - ) - - # Check if past and future covariates have the same dimension - past_cov_dim = past_covariates.shape[1] - future_cov_dim = future_covariates.shape[1] - - if future_cov_dim == 0: - raise_log( - ValueError( - "ExoMamba requires future covariates with the same dimension as past covariates." - ), - logger=logger, - ) - - # Use the shared dimension for dynamic covariates - dynamic_cov_dim = future_cov_dim - - # Input dimension includes target and covariates - input_dim = dynamic_cov_dim - - # Handle static covariates properly - if static_covariates is not None: - # Flatten if it has more than 2 dimensions - if len(static_covariates.shape) > 2: - static_cov_dim = ( - static_covariates.shape[0] * static_covariates.shape[-1] - ) - logger.warning( - f"Static covariates have unexpected shape {static_covariates.shape}. " - f"Flattening to dimension {static_cov_dim}." - ) - else: - static_cov_dim = static_covariates.shape[-1] - else: - static_cov_dim = 0 - - # Number of parameters (1 for point forecasts, more for probabilistic forecasts) - nr_params = 1 if self.likelihood is None else self.likelihood.num_parameters - - # Create model - model = _ExoMambaModule( - input_dim=input_dim, - output_dim=output_dim, - dynamic_cov_dim=dynamic_cov_dim, - static_cov_dim=static_cov_dim, - nr_params=nr_params, - hidden_size=self.hidden_size, - n_mamba_layers=self.n_mamba_layers, - d_state=self.d_state, - expand_factor=self.expand_factor, - dropout=self.dropout, - **self.pl_module_params, - ) - - return model - - @property - def supports_static_covariates(self) -> bool: - return True - - @property - def supports_multivariate(self) -> bool: - return True - - @property - def ignores_past_target_values(self) -> bool: - """Indicates that this model does not use target's past values for prediction.""" - return True diff --git a/st_forecast/custom_models/lstm/__init__.py b/st_forecast/custom_models/lstm/__init__.py deleted file mode 100644 index c79f334..0000000 --- a/st_forecast/custom_models/lstm/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from st_forecast.custom_models.lstm.model import LSTMModel - -__all__ = ["LSTMModel"] diff --git a/st_forecast/custom_models/lstm/model.py b/st_forecast/custom_models/lstm/model.py deleted file mode 100644 index 75b5b2f..0000000 --- a/st_forecast/custom_models/lstm/model.py +++ /dev/null @@ -1,99 +0,0 @@ -import torch - -from darts.logging import get_logger -from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel - -from st_forecast.custom_models.lstm.module import _LSTMModule - -MixedCovariatesTrainTensorType = tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor -] - -logger = get_logger(__name__) - - -class LSTMModel(MixedCovariatesTorchModel): - def __init__( - self, - input_chunk_length: int, - output_chunk_length: int, - output_chunk_shift: int = 0, - hidden_size: int = 64, - n_lstm_layers: int = 2, - dropout: float = 0.1, - use_past_target: bool = True, - use_learned_init: bool = True, - use_static_covariates: bool = True, - obs_mask_prob: float = 0.0, - **kwargs, - ): - super().__init__(**self._extract_torch_model_params(**self.model_params)) - - self._considers_static_covariates = use_static_covariates - - self.pl_module_params = self._extract_pl_module_params(**self.model_params) - - self.hidden_size = hidden_size - self.n_lstm_layers = n_lstm_layers - self.dropout = dropout - self.use_past_target = use_past_target - self.use_learned_init = use_learned_init - self.obs_mask_prob = obs_mask_prob - - def _create_model( - self, train_sample: MixedCovariatesTrainTensorType - ) -> torch.nn.Module: - ( - past_target, - past_covariates, - historic_future_covariates, - future_covariates, - static_covariates, - future_target, - ) = train_sample - - output_dim = future_target.shape[1] - - past_cov_dim = past_covariates.shape[1] if past_covariates is not None else 0 - future_cov_dim = ( - future_covariates.shape[1] if future_covariates is not None else 0 - ) - - if static_covariates is not None: - if len(static_covariates.shape) > 2: - static_cov_dim = ( - static_covariates.shape[0] * static_covariates.shape[-1] - ) - else: - static_cov_dim = static_covariates.shape[-1] - else: - static_cov_dim = 0 - - nr_params = 1 if self.likelihood is None else self.likelihood.num_parameters - - return _LSTMModule( - output_dim=output_dim, - past_cov_dim=past_cov_dim, - future_cov_dim=future_cov_dim, - static_cov_dim=static_cov_dim, - nr_params=nr_params, - hidden_size=self.hidden_size, - n_lstm_layers=self.n_lstm_layers, - dropout=self.dropout, - use_past_target=self.use_past_target, - use_learned_init=self.use_learned_init, - obs_mask_prob=self.obs_mask_prob, - **self.pl_module_params, - ) - - @property - def supports_static_covariates(self) -> bool: - return True - - @property - def supports_multivariate(self) -> bool: - return True - - @property - def ignores_past_target_values(self) -> bool: - return not self.use_past_target diff --git a/st_forecast/custom_models/lstm/module.py b/st_forecast/custom_models/lstm/module.py deleted file mode 100644 index 45e9608..0000000 --- a/st_forecast/custom_models/lstm/module.py +++ /dev/null @@ -1,207 +0,0 @@ -import torch -import torch.nn as nn - -from darts.logging import get_logger -from darts.models.forecasting.pl_forecasting_module import ( - PLForecastingModule, - io_processor, -) -from darts.utils.torch import MonteCarloDropout - -logger = get_logger(__name__) - - -class _LSTMModule(PLForecastingModule): - def __init__( - self, - output_dim: int, - past_cov_dim: int, - future_cov_dim: int, - static_cov_dim: int, - nr_params: int, - hidden_size: int, - n_lstm_layers: int, - dropout: float, - use_past_target: bool = True, - use_learned_init: bool = True, - obs_mask_prob: float = 0.0, - **kwargs, - ): - super().__init__(**kwargs) - - self.output_dim = output_dim - self.obs_mask_prob = obs_mask_prob - self.past_cov_dim = past_cov_dim - self.future_cov_dim = future_cov_dim - self.static_cov_dim = static_cov_dim - self.nr_params = nr_params - self.hidden_size = hidden_size - self.n_lstm_layers = n_lstm_layers - self.use_past_target = use_past_target - - # Past input dim = past_target (if used) + past_cov - past_input_dim = past_cov_dim - if use_past_target: - past_input_dim += output_dim - - # Indicator channel: 1 if any past-only features exist - self.has_indicator = past_input_dim > 0 - indicator_dim = 1 if self.has_indicator else 0 - - # combined_dim = static + past_input + indicator + future_cov - combined_dim = static_cov_dim + past_input_dim + indicator_dim + future_cov_dim - self.past_input_dim = past_input_dim - - self.activation = nn.ReLU() - self.dropout_layer = MonteCarloDropout(dropout) - - self.input_proj = nn.Linear(combined_dim, hidden_size) - self.lstm = nn.LSTM( - input_size=hidden_size, - hidden_size=hidden_size, - num_layers=n_lstm_layers, - dropout=dropout if n_lstm_layers > 1 else 0, - batch_first=True, - ) - self.output_proj = nn.Linear(hidden_size, output_dim * nr_params) - - # Learned hidden state initialization from static covariates - if use_learned_init and static_cov_dim > 0: - self.hidden_init = nn.Sequential( - nn.Linear(static_cov_dim, hidden_size), - nn.SiLU(), - nn.Linear(hidden_size, hidden_size * n_lstm_layers), - ) - self.cell_init = nn.Sequential( - nn.Linear(static_cov_dim, hidden_size), - nn.SiLU(), - nn.Linear(hidden_size, hidden_size * n_lstm_layers), - ) - else: - self.hidden_init = None - self.cell_init = None - - def _init_hidden_state( - self, x_static_covariates: torch.Tensor | None, batch_size: int - ) -> tuple[torch.Tensor, torch.Tensor] | None: - if self.hidden_init is None or x_static_covariates is None: - return None - - static_flat = x_static_covariates.view(batch_size, -1) - h_0 = self.hidden_init(static_flat) - c_0 = self.cell_init(static_flat) - - h_0 = h_0.view(self.n_lstm_layers, batch_size, self.hidden_size) - c_0 = c_0.view(self.n_lstm_layers, batch_size, self.hidden_size) - - return h_0, c_0 - - @io_processor - def forward( - self, x_in: tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None] - ) -> torch.Tensor: - x, x_future_covariates, x_static_covariates = x_in - batch_size = x.shape[0] - past_len = x.shape[1] - future_len = ( - x_future_covariates.shape[1] - if x_future_covariates is not None - else self.output_chunk_length - ) - total_len = past_len + future_len - - # Flatten static covariates - if x_static_covariates is not None and len(x_static_covariates.shape) > 2: - x_static_covariates = x_static_covariates.view(batch_size, -1) - - parts = [] - keep_mask = None - - # Static covariates → broadcast to [B, L+T, S] - if self.static_cov_dim > 0 and x_static_covariates is not None: - static_expanded = x_static_covariates.unsqueeze(1).expand( - batch_size, total_len, -1 - ) - parts.append(static_expanded) - - # Past-only features (past_target + past_cov) → zero-padded to [B, L+T, past_input_dim] - if self.past_input_dim > 0: - past_parts = [] - if self.use_past_target: - past_parts.append(x[:, :, : self.output_dim]) - if self.past_cov_dim > 0: - past_parts.append( - x[:, :, self.output_dim : self.output_dim + self.past_cov_dim] - ) - past_features = torch.cat(past_parts, dim=2) - - # Random observation masking during training - if self.training and self.obs_mask_prob > 0.0 and self.use_past_target: - keep_mask = ( - torch.rand(batch_size, past_len, 1, device=x.device) - >= self.obs_mask_prob - ) - past_features = torch.cat( - [ - past_features[:, :, : self.output_dim] * keep_mask, - past_features[:, :, self.output_dim :], - ], - dim=2, - ) - - # Zero-pad into future - extended_past = torch.zeros( - batch_size, - total_len, - self.past_input_dim, - dtype=x.dtype, - device=x.device, - ) - extended_past[:, :past_len, :] = past_features - parts.append(extended_past) - - # Indicator channel: 1=observed, 0=masked - if self.has_indicator: - indicator = torch.zeros( - batch_size, total_len, 1, dtype=x.dtype, device=x.device - ) - indicator[:, :past_len, :] = ( - keep_mask.float() if keep_mask is not None else 1.0 - ) - parts.append(indicator) - - # Future covariates → zero-padded to [B, L+T, F] - if x_future_covariates is not None and self.future_cov_dim > 0: - padded_future = torch.zeros( - batch_size, - total_len, - self.future_cov_dim, - dtype=x.dtype, - device=x.device, - ) - padded_future[:, past_len:, :] = x_future_covariates - parts.append(padded_future) - - # Concatenate → [B, L+T, combined_dim] - combined = torch.cat(parts, dim=2) - - # Project → ReLU → Dropout - projected = self.dropout_layer(self.activation(self.input_proj(combined))) - - # LSTM - init_state = self._init_hidden_state(x_static_covariates, batch_size) - if init_state is not None: - lstm_out, _ = self.lstm(projected, init_state) - else: - lstm_out, _ = self.lstm(projected) - - # Slice last T timesteps - forecast_out = lstm_out[:, past_len:, :] - - # Output projection → [B, T, output_dim, nr_params] - output = self.output_proj(forecast_out) - output = output.view( - batch_size, self.output_chunk_length, self.output_dim, self.nr_params - ) - - return output diff --git a/st_forecast/custom_models/lstm_enc_dec/__init__.py b/st_forecast/custom_models/lstm_enc_dec/__init__.py deleted file mode 100644 index 17db528..0000000 --- a/st_forecast/custom_models/lstm_enc_dec/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from st_forecast.custom_models.lstm_enc_dec.model import LSTMEncoderDecoderModel - -__all__ = ["LSTMEncoderDecoderModel"] diff --git a/st_forecast/custom_models/lstm_enc_dec/model.py b/st_forecast/custom_models/lstm_enc_dec/model.py deleted file mode 100644 index db29eb4..0000000 --- a/st_forecast/custom_models/lstm_enc_dec/model.py +++ /dev/null @@ -1,103 +0,0 @@ -import torch - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel - -from st_forecast.custom_models.lstm_enc_dec.module import _LSTMEncoderDecoderModule - -MixedCovariatesTrainTensorType = tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor -] - -logger = get_logger(__name__) - - -class LSTMEncoderDecoderModel(MixedCovariatesTorchModel): - def __init__( - self, - input_chunk_length: int, - output_chunk_length: int, - output_chunk_shift: int = 0, - hidden_size: int = 64, - n_layers: int = 2, - dropout: float = 0.1, - use_past_target: bool = True, - use_learned_init: bool = True, - use_skip_connection: bool = True, - use_static_covariates: bool = True, - **kwargs, - ): - super().__init__(**self._extract_torch_model_params(**self.model_params)) - - self._considers_static_covariates = use_static_covariates - - self.pl_module_params = self._extract_pl_module_params(**self.model_params) - - self.hidden_size = hidden_size - self.n_layers = n_layers - self.dropout = dropout - self.use_past_target = use_past_target - self.use_learned_init = use_learned_init - self.use_skip_connection = use_skip_connection - - def _create_model( - self, train_sample: MixedCovariatesTrainTensorType - ) -> torch.nn.Module: - ( - past_target, - past_covariates, - historic_future_covariates, - future_covariates, - static_covariates, - future_target, - ) = train_sample - - output_dim = future_target.shape[1] - - if future_covariates is None: - raise_log( - ValueError("LSTMEncoderDecoderModel requires future covariates."), - logger=logger, - ) - - past_cov_dim = past_covariates.shape[1] if past_covariates is not None else 0 - future_cov_dim = future_covariates.shape[1] - - if static_covariates is not None: - if len(static_covariates.shape) > 2: - static_cov_dim = ( - static_covariates.shape[0] * static_covariates.shape[-1] - ) - else: - static_cov_dim = static_covariates.shape[-1] - else: - static_cov_dim = 0 - - nr_params = 1 if self.likelihood is None else self.likelihood.num_parameters - - return _LSTMEncoderDecoderModule( - output_dim=output_dim, - past_cov_dim=past_cov_dim, - future_cov_dim=future_cov_dim, - static_cov_dim=static_cov_dim, - nr_params=nr_params, - hidden_size=self.hidden_size, - n_layers=self.n_layers, - dropout=self.dropout, - use_past_target=self.use_past_target, - use_learned_init=self.use_learned_init, - use_skip_connection=self.use_skip_connection, - **self.pl_module_params, - ) - - @property - def supports_static_covariates(self) -> bool: - return True - - @property - def supports_multivariate(self) -> bool: - return True - - @property - def ignores_past_target_values(self) -> bool: - return not self.use_past_target diff --git a/st_forecast/custom_models/lstm_enc_dec/module.py b/st_forecast/custom_models/lstm_enc_dec/module.py deleted file mode 100644 index ec1a7b2..0000000 --- a/st_forecast/custom_models/lstm_enc_dec/module.py +++ /dev/null @@ -1,248 +0,0 @@ -import torch -import torch.nn as nn - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.pl_forecasting_module import ( - PLForecastingModule, - io_processor, -) -from darts.utils.torch import MonteCarloDropout - -logger = get_logger(__name__) - - -class _LSTMEncoderDecoderModule(PLForecastingModule): - def __init__( - self, - output_dim: int, - past_cov_dim: int, - future_cov_dim: int, - static_cov_dim: int, - nr_params: int, - hidden_size: int, - n_layers: int, - dropout: float, - use_past_target: bool = True, - use_learned_init: bool = True, - use_skip_connection: bool = True, - **kwargs, - ): - super().__init__(**kwargs) - - self.output_dim = output_dim - self.past_cov_dim = past_cov_dim - self.future_cov_dim = future_cov_dim - self.static_cov_dim = static_cov_dim - self.nr_params = nr_params - self.hidden_size = hidden_size - self.n_layers = n_layers - self.dropout = dropout - self.use_past_target = use_past_target - self.use_learned_init = use_learned_init - self.use_skip_connection = use_skip_connection - - # Determine if encoder is needed (has temporal info to encode) - self.needs_encoder = use_past_target or past_cov_dim > 0 - - # Decoder input: [static | future_cov] - decoder_input_dim = static_cov_dim + future_cov_dim - - self.activation = nn.SiLU() - self.dropout_layer = MonteCarloDropout(dropout) - - # Encoder layers (only if needed) - if self.needs_encoder: - # Encoder input: [static | past_target (optional) | past_cov] - encoder_input_dim = static_cov_dim + past_cov_dim - if use_past_target: - encoder_input_dim += output_dim - - self.encoder_proj = nn.Linear(encoder_input_dim, hidden_size) - self.encoder_lstm = nn.LSTM( - input_size=hidden_size, - hidden_size=hidden_size, - num_layers=n_layers, - dropout=dropout if n_layers > 1 else 0, - batch_first=True, - ) - else: - self.encoder_proj = None - self.encoder_lstm = None - - self.decoder_proj = nn.Linear(decoder_input_dim, hidden_size) - self.decoder_lstm = nn.LSTM( - input_size=hidden_size, - hidden_size=hidden_size, - num_layers=n_layers, - dropout=dropout if n_layers > 1 else 0, - batch_first=True, - ) - - self.output_proj = nn.Linear(hidden_size, output_dim * nr_params) - - # Learned hidden state initialization from static covariates - if use_learned_init and static_cov_dim > 0: - self.hidden_init = nn.Sequential( - nn.Linear(static_cov_dim, hidden_size), - nn.SiLU(), - nn.Linear(hidden_size, hidden_size * n_layers), - ) - self.cell_init = nn.Sequential( - nn.Linear(static_cov_dim, hidden_size), - nn.SiLU(), - nn.Linear(hidden_size, hidden_size * n_layers), - ) - else: - self.hidden_init = None - self.cell_init = None - - # Gated skip connections (residual) - if use_skip_connection: - if self.needs_encoder: - self.encoder_gate = nn.Linear(hidden_size * 2, hidden_size) - else: - self.encoder_gate = None - self.decoder_gate = nn.Linear(hidden_size * 2, hidden_size) - else: - self.encoder_gate = None - self.decoder_gate = None - - def _init_hidden_state( - self, x_static_covariates: torch.Tensor | None, batch_size: int - ) -> tuple[torch.Tensor, torch.Tensor] | None: - """Initialize LSTM hidden state from static covariates.""" - if self.hidden_init is None or x_static_covariates is None: - return None - - static_flat = x_static_covariates.view(batch_size, -1) - h_0 = self.hidden_init(static_flat) - c_0 = self.cell_init(static_flat) - - # Reshape to (n_layers, batch_size, hidden_size) - h_0 = h_0.view(self.n_layers, batch_size, self.hidden_size) - c_0 = c_0.view(self.n_layers, batch_size, self.hidden_size) - - return h_0, c_0 - - def _apply_skip_connection( - self, - output: torch.Tensor, - residual: torch.Tensor, - gate_layer: nn.Linear | None, - ) -> torch.Tensor: - """Apply gated skip connection.""" - if gate_layer is None: - return output - - gate_input = torch.cat([output, residual], dim=-1) - gate = torch.sigmoid(gate_layer(gate_input)) - return gate * output + (1 - gate) * residual - - @io_processor - def forward( - self, x_in: tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None] - ) -> torch.Tensor: - x, x_future_covariates, x_static_covariates = x_in - batch_size = x.shape[0] - past_seq_len = x.shape[1] - - if x_future_covariates is None: - raise_log( - ValueError("LSTMEncoderDecoder requires future covariates."), - logger=logger, - ) - - future_seq_len = x_future_covariates.shape[1] - - # Flatten static covariates if needed - if x_static_covariates is not None and len(x_static_covariates.shape) > 2: - x_static_covariates = x_static_covariates.view(batch_size, -1) - - # Build decoder input: [static | future_cov] - decoder_parts = [] - - if self.static_cov_dim > 0 and x_static_covariates is not None: - static_future = x_static_covariates.unsqueeze(1).expand( - batch_size, future_seq_len, -1 - ) - decoder_parts.append(static_future) - - decoder_parts.append(x_future_covariates) - decoder_input = torch.cat(decoder_parts, dim=2) - - # Encoder (only if needed) - if self.needs_encoder: - # Build encoder input parts - encoder_parts = [] - - # Static covariates (expanded to past sequence length) - if self.static_cov_dim > 0 and x_static_covariates is not None: - static_past = x_static_covariates.unsqueeze(1).expand( - batch_size, past_seq_len, -1 - ) - encoder_parts.append(static_past) - - # Past target (optional) - if self.use_past_target: - x_past_target = x[:, :, : self.output_dim] - encoder_parts.append(x_past_target) - - # Past covariates - if self.past_cov_dim > 0: - x_past_cov = x[ - :, :, self.output_dim : self.output_dim + self.past_cov_dim - ] - encoder_parts.append(x_past_cov) - - encoder_input = torch.cat(encoder_parts, dim=2) - - # Encoder forward pass - encoder_embedded = self.dropout_layer( - self.activation(self.encoder_proj(encoder_input)) - ) - - # Initialize hidden state from static covariates (or use zeros) - init_state = self._init_hidden_state(x_static_covariates, batch_size) - if init_state is not None: - encoder_output, (h_n, c_n) = self.encoder_lstm( - encoder_embedded, init_state - ) - else: - encoder_output, (h_n, c_n) = self.encoder_lstm(encoder_embedded) - - """# Apply skip connection to encoder output - encoder_output = self._apply_skip_connection( - encoder_output, encoder_embedded, self.encoder_gate - )""" - else: - # No encoder - initialize decoder directly from static covariates or zeros - init_state = self._init_hidden_state(x_static_covariates, batch_size) - if init_state is not None: - h_n, c_n = init_state - else: - # Use zeros for hidden state when no static covariates - h_n = torch.zeros( - self.n_layers, batch_size, self.hidden_size, device=x.device - ) - c_n = torch.zeros( - self.n_layers, batch_size, self.hidden_size, device=x.device - ) - - # Decoder (initialized with encoder hidden state or learned init) - decoder_embedded = self.dropout_layer( - self.activation(self.decoder_proj(decoder_input)) - ) - decoder_output, _ = self.decoder_lstm(decoder_embedded, (h_n, c_n)) - - # Apply skip connection to decoder output - """decoder_output = self._apply_skip_connection( - decoder_output, decoder_embedded, self.decoder_gate - )""" - - # Output projection - output = self.output_proj(decoder_output) - output = output.view( - batch_size, self.output_chunk_length, self.output_dim, self.nr_params - ) - - return output diff --git a/st_forecast/custom_models/lstm_enc_mlp/__init__.py b/st_forecast/custom_models/lstm_enc_mlp/__init__.py deleted file mode 100644 index d806358..0000000 --- a/st_forecast/custom_models/lstm_enc_mlp/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from st_forecast.custom_models.lstm_enc_mlp.model import LSTMEncMLPModel - -__all__ = ["LSTMEncMLPModel"] diff --git a/st_forecast/custom_models/lstm_enc_mlp/model.py b/st_forecast/custom_models/lstm_enc_mlp/model.py deleted file mode 100644 index 5b598a2..0000000 --- a/st_forecast/custom_models/lstm_enc_mlp/model.py +++ /dev/null @@ -1,109 +0,0 @@ -import torch - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel - -from st_forecast.custom_models.lstm_enc_mlp.module import _LSTMEncMLPModule - -MixedCovariatesTrainTensorType = tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor -] - -logger = get_logger(__name__) - - -class LSTMEncMLPModel(MixedCovariatesTorchModel): - def __init__( - self, - input_chunk_length: int, - output_chunk_length: int, - output_chunk_shift: int = 0, - hidden_size: int = 64, - n_layers: int = 2, - dropout: float = 0.1, - use_past_target: bool = True, - use_learned_init: bool = True, - num_decoder_blocks: int = 2, - decoder_activation: str = "ReLU", - use_layer_norm: bool = False, - use_static_covariates: bool = True, - **kwargs, - ): - super().__init__(**self._extract_torch_model_params(**self.model_params)) - - self._considers_static_covariates = use_static_covariates - - self.pl_module_params = self._extract_pl_module_params(**self.model_params) - - self.hidden_size = hidden_size - self.n_layers = n_layers - self.dropout = dropout - self.use_past_target = use_past_target - self.use_learned_init = use_learned_init - self.num_decoder_blocks = num_decoder_blocks - self.decoder_activation = decoder_activation - self.use_layer_norm = use_layer_norm - - def _create_model( - self, train_sample: MixedCovariatesTrainTensorType - ) -> torch.nn.Module: - ( - past_target, - past_covariates, - historic_future_covariates, - future_covariates, - static_covariates, - future_target, - ) = train_sample - - output_dim = future_target.shape[1] - - if future_covariates is None: - raise_log( - ValueError("LSTMEncMLPModel requires future covariates."), - logger=logger, - ) - - past_cov_dim = past_covariates.shape[1] if past_covariates is not None else 0 - future_cov_dim = future_covariates.shape[1] - - if static_covariates is not None: - if len(static_covariates.shape) > 2: - static_cov_dim = ( - static_covariates.shape[0] * static_covariates.shape[-1] - ) - else: - static_cov_dim = static_covariates.shape[-1] - else: - static_cov_dim = 0 - - nr_params = 1 if self.likelihood is None else self.likelihood.num_parameters - - return _LSTMEncMLPModule( - output_dim=output_dim, - past_cov_dim=past_cov_dim, - future_cov_dim=future_cov_dim, - static_cov_dim=static_cov_dim, - nr_params=nr_params, - hidden_size=self.hidden_size, - n_layers=self.n_layers, - dropout=self.dropout, - use_past_target=self.use_past_target, - use_learned_init=self.use_learned_init, - num_decoder_blocks=self.num_decoder_blocks, - decoder_activation=self.decoder_activation, - use_layer_norm=self.use_layer_norm, - **self.pl_module_params, - ) - - @property - def supports_static_covariates(self) -> bool: - return True - - @property - def supports_multivariate(self) -> bool: - return True - - @property - def ignores_past_target_values(self) -> bool: - return not self.use_past_target diff --git a/st_forecast/custom_models/lstm_enc_mlp/module.py b/st_forecast/custom_models/lstm_enc_mlp/module.py deleted file mode 100644 index 3e09d67..0000000 --- a/st_forecast/custom_models/lstm_enc_mlp/module.py +++ /dev/null @@ -1,267 +0,0 @@ -import torch -import torch.nn as nn - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.pl_forecasting_module import ( - PLForecastingModule, - io_processor, -) -from darts.utils.torch import MonteCarloDropout - -logger = get_logger(__name__) - -ACTIVATIONS: dict[str, type[nn.Module]] = { - "ReLU": nn.ReLU, - "SiLU": nn.SiLU, - "GELU": nn.GELU, - "Tanh": nn.Tanh, -} - - -class _ResidualBlock(nn.Module): - def __init__( - self, - input_dim: int, - output_dim: int, - hidden_size: int, - dropout: float, - activation: str = "ReLU", - use_layer_norm: bool = False, - ): - super().__init__() - act_cls = ACTIVATIONS.get(activation, nn.ReLU) - - self.dense = nn.Sequential( - nn.Linear(input_dim, hidden_size), - act_cls(), - nn.Linear(hidden_size, output_dim), - MonteCarloDropout(dropout), - ) - self.skip = nn.Linear(input_dim, output_dim) - self.layer_norm = nn.LayerNorm(output_dim) if use_layer_norm else None - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = self.dense(x) + self.skip(x) - if self.layer_norm is not None: - out = self.layer_norm(out) - return out - - -class _LSTMEncMLPModule(PLForecastingModule): - def __init__( - self, - output_dim: int, - past_cov_dim: int, - future_cov_dim: int, - static_cov_dim: int, - nr_params: int, - hidden_size: int, - n_layers: int, - dropout: float, - use_past_target: bool = True, - use_learned_init: bool = True, - num_decoder_blocks: int = 2, - decoder_activation: str = "ReLU", - use_layer_norm: bool = False, - **kwargs, - ): - super().__init__(**kwargs) - - self.output_dim = output_dim - self.past_cov_dim = past_cov_dim - self.future_cov_dim = future_cov_dim - self.static_cov_dim = static_cov_dim - self.nr_params = nr_params - self.hidden_size = hidden_size - self.n_layers = n_layers - self.dropout = dropout - self.use_past_target = use_past_target - self.use_learned_init = use_learned_init - - self.needs_encoder = use_past_target or past_cov_dim > 0 - - activation = nn.ReLU() - self.dropout_layer = MonteCarloDropout(dropout) - - # --- Encoder (identical to LSTMEncDec) --- - if self.needs_encoder: - encoder_input_dim = static_cov_dim + past_cov_dim - if use_past_target: - encoder_input_dim += output_dim - - self.encoder_proj = nn.Linear(encoder_input_dim, hidden_size) - self.encoder_activation = activation - self.encoder_lstm = nn.LSTM( - input_size=hidden_size, - hidden_size=hidden_size, - num_layers=n_layers, - dropout=dropout if n_layers > 1 else 0, - batch_first=True, - ) - else: - self.encoder_proj = None - self.encoder_lstm = None - self.encoder_activation = None - - # Learned hidden state initialization from static covariates - if use_learned_init and static_cov_dim > 0: - self.hidden_init = nn.Sequential( - nn.Linear(static_cov_dim, hidden_size), - nn.SiLU(), - nn.Linear(hidden_size, hidden_size * n_layers), - ) - self.cell_init = nn.Sequential( - nn.Linear(static_cov_dim, hidden_size), - nn.SiLU(), - nn.Linear(hidden_size, hidden_size * n_layers), - ) - else: - self.hidden_init = None - self.cell_init = None - - # --- MLP Decoder (residual block stack) --- - # Input: [h_n[-1] | flatten(future_covariates) | static_covariates] - context_dim = hidden_size if self.needs_encoder else 0 - mlp_input_dim = ( - context_dim + future_cov_dim * self.output_chunk_length + static_cov_dim - ) - - final_output_dim = self.output_chunk_length * output_dim * nr_params - - num_decoder_blocks = max(num_decoder_blocks, 1) - blocks: list[nn.Module] = [] - - if num_decoder_blocks == 1: - blocks.append( - _ResidualBlock( - mlp_input_dim, - final_output_dim, - hidden_size, - dropout, - decoder_activation, - use_layer_norm, - ) - ) - else: - blocks.append( - _ResidualBlock( - mlp_input_dim, - hidden_size, - hidden_size, - dropout, - decoder_activation, - use_layer_norm, - ) - ) - for _ in range(num_decoder_blocks - 2): - blocks.append( - _ResidualBlock( - hidden_size, - hidden_size, - hidden_size, - dropout, - decoder_activation, - use_layer_norm, - ) - ) - blocks.append( - _ResidualBlock( - hidden_size, - final_output_dim, - hidden_size, - dropout, - decoder_activation, - use_layer_norm, - ) - ) - - self.decoder_blocks = nn.ModuleList(blocks) - - def _init_hidden_state( - self, x_static_covariates: torch.Tensor | None, batch_size: int - ) -> tuple[torch.Tensor, torch.Tensor] | None: - if self.hidden_init is None or x_static_covariates is None: - return None - - static_flat = x_static_covariates.view(batch_size, -1) - h_0 = self.hidden_init(static_flat).view( - self.n_layers, batch_size, self.hidden_size - ) - c_0 = self.cell_init(static_flat).view( - self.n_layers, batch_size, self.hidden_size - ) - return h_0, c_0 - - @io_processor - def forward( - self, x_in: tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None] - ) -> torch.Tensor: - x, x_future_covariates, x_static_covariates = x_in - batch_size = x.shape[0] - past_seq_len = x.shape[1] - - if x_future_covariates is None: - raise_log( - ValueError("LSTMEncMLP requires future covariates."), - logger=logger, - ) - - # Flatten static covariates - if x_static_covariates is not None and len(x_static_covariates.shape) > 2: - x_static_covariates = x_static_covariates.view(batch_size, -1) - - # --- Encoder --- - if self.needs_encoder: - encoder_parts = [] - - if self.static_cov_dim > 0 and x_static_covariates is not None: - static_past = x_static_covariates.unsqueeze(1).expand( - batch_size, past_seq_len, -1 - ) - encoder_parts.append(static_past) - - if self.use_past_target: - encoder_parts.append(x[:, :, : self.output_dim]) - - if self.past_cov_dim > 0: - encoder_parts.append( - x[:, :, self.output_dim : self.output_dim + self.past_cov_dim] - ) - - encoder_input = torch.cat(encoder_parts, dim=2) - encoder_embedded = self.dropout_layer( - self.encoder_activation(self.encoder_proj(encoder_input)) - ) - - init_state = self._init_hidden_state(x_static_covariates, batch_size) - if init_state is not None: - _, (h_n, _) = self.encoder_lstm(encoder_embedded, init_state) - else: - _, (h_n, _) = self.encoder_lstm(encoder_embedded) - - context = h_n[-1] # (batch, hidden_size) - - # --- Build MLP decoder input --- - mlp_parts = [] - - if self.needs_encoder: - mlp_parts.append(context) - - # Flatten future covariates: (batch, output_chunk_length, future_cov_dim) → (batch, output_chunk_length * future_cov_dim) - mlp_parts.append(x_future_covariates.reshape(batch_size, -1)) - - if self.static_cov_dim > 0 and x_static_covariates is not None: - mlp_parts.append(x_static_covariates) - - mlp_input = torch.cat(mlp_parts, dim=1) - - # --- Pass through residual blocks --- - h = mlp_input - for block in self.decoder_blocks: - h = block(h) - - # Reshape to (batch, output_chunk_length, output_dim, nr_params) - output = h.view( - batch_size, self.output_chunk_length, self.output_dim, self.nr_params - ) - return output diff --git a/st_forecast/custom_models/mamba_enc_dec/__init__.py b/st_forecast/custom_models/mamba_enc_dec/__init__.py deleted file mode 100644 index 11fae65..0000000 --- a/st_forecast/custom_models/mamba_enc_dec/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from st_forecast.custom_models.mamba_enc_dec.model import MambaEncoderDecoderModel - -__all__ = ["MambaEncoderDecoderModel"] diff --git a/st_forecast/custom_models/mamba_enc_dec/model.py b/st_forecast/custom_models/mamba_enc_dec/model.py deleted file mode 100644 index a055846..0000000 --- a/st_forecast/custom_models/mamba_enc_dec/model.py +++ /dev/null @@ -1,103 +0,0 @@ -import torch - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel - -from st_forecast.custom_models.mamba_enc_dec.module import _MambaEncoderDecoderModule - -MixedCovariatesTrainTensorType = tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor -] - -logger = get_logger(__name__) - - -class MambaEncoderDecoderModel(MixedCovariatesTorchModel): - def __init__( - self, - input_chunk_length: int, - output_chunk_length: int, - output_chunk_shift: int = 0, - hidden_size: int = 64, - n_layers: int = 2, - d_state: int = 16, - expand_factor: int = 2, - dropout: float = 0.1, - use_past_target: bool = True, - use_static_covariates: bool = True, - **kwargs, - ): - super().__init__(**self._extract_torch_model_params(**self.model_params)) - - self._considers_static_covariates = use_static_covariates - - self.pl_module_params = self._extract_pl_module_params(**self.model_params) - - self.hidden_size = hidden_size - self.n_layers = n_layers - self.d_state = d_state - self.expand_factor = expand_factor - self.dropout = dropout - self.use_past_target = use_past_target - - def _create_model( - self, train_sample: MixedCovariatesTrainTensorType - ) -> torch.nn.Module: - ( - past_target, - past_covariates, - historic_future_covariates, - future_covariates, - static_covariates, - future_target, - ) = train_sample - - output_dim = future_target.shape[1] - - if future_covariates is None: - raise_log( - ValueError("MambaEncoderDecoderModel requires future covariates."), - logger=logger, - ) - - past_cov_dim = past_covariates.shape[1] if past_covariates is not None else 0 - future_cov_dim = future_covariates.shape[1] - - if static_covariates is not None: - if len(static_covariates.shape) > 2: - static_cov_dim = ( - static_covariates.shape[0] * static_covariates.shape[-1] - ) - else: - static_cov_dim = static_covariates.shape[-1] - else: - static_cov_dim = 0 - - nr_params = 1 if self.likelihood is None else self.likelihood.num_parameters - - return _MambaEncoderDecoderModule( - output_dim=output_dim, - past_cov_dim=past_cov_dim, - future_cov_dim=future_cov_dim, - static_cov_dim=static_cov_dim, - nr_params=nr_params, - hidden_size=self.hidden_size, - n_layers=self.n_layers, - d_state=self.d_state, - expand_factor=self.expand_factor, - dropout=self.dropout, - use_past_target=self.use_past_target, - **self.pl_module_params, - ) - - @property - def supports_static_covariates(self) -> bool: - return True - - @property - def supports_multivariate(self) -> bool: - return True - - @property - def ignores_past_target_values(self) -> bool: - return not self.use_past_target diff --git a/st_forecast/custom_models/mamba_enc_dec/module.py b/st_forecast/custom_models/mamba_enc_dec/module.py deleted file mode 100644 index 8e02c8b..0000000 --- a/st_forecast/custom_models/mamba_enc_dec/module.py +++ /dev/null @@ -1,167 +0,0 @@ -import torch -import torch.nn as nn - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.pl_forecasting_module import ( - PLForecastingModule, - io_processor, -) -from darts.utils.torch import MonteCarloDropout - -try: - from mambapy.mamba import Mamba, MambaConfig -except ImportError: - raise ImportError( - "Mamba is not installed. Please install it using `pip install mambapy`." - ) - -logger = get_logger(__name__) - - -class _MambaEncoderDecoderModule(PLForecastingModule): - def __init__( - self, - output_dim: int, - past_cov_dim: int, - future_cov_dim: int, - static_cov_dim: int, - nr_params: int, - hidden_size: int, - n_layers: int, - d_state: int, - expand_factor: int, - dropout: float, - use_past_target: bool = True, - **kwargs, - ): - super().__init__(**kwargs) - - self.output_dim = output_dim - self.past_cov_dim = past_cov_dim - self.future_cov_dim = future_cov_dim - self.static_cov_dim = static_cov_dim - self.nr_params = nr_params - self.hidden_size = hidden_size - self.n_layers = n_layers - self.d_state = d_state - self.expand_factor = expand_factor - self.dropout = dropout - self.use_past_target = use_past_target - - # Encoder input: [static | past_target (optional) | past_cov] - encoder_input_dim = static_cov_dim + past_cov_dim - if use_past_target: - encoder_input_dim += output_dim - - # Decoder input: [static | future_cov] - decoder_input_dim = static_cov_dim + future_cov_dim - - self.encoder_proj = nn.Linear(encoder_input_dim, hidden_size) - self.decoder_proj = nn.Linear(decoder_input_dim, hidden_size) - - self.activation = nn.SiLU() - self.dropout_layer = MonteCarloDropout(dropout) - - encoder_config = MambaConfig( - d_model=hidden_size, - n_layers=n_layers, - d_state=d_state, - expand_factor=expand_factor, - ) - self.encoder_mamba = Mamba(encoder_config) - - decoder_config = MambaConfig( - d_model=hidden_size, - n_layers=n_layers, - d_state=d_state, - expand_factor=expand_factor, - ) - self.decoder_mamba = Mamba(decoder_config) - - self.context_proj = nn.Sequential( - nn.Linear(hidden_size, hidden_size), - nn.SiLU(), - nn.Linear(hidden_size, hidden_size), - ) - - self.output_proj = nn.Linear(hidden_size, output_dim * nr_params) - - @io_processor - def forward( - self, x_in: tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None] - ) -> torch.Tensor: - x, x_future_covariates, x_static_covariates = x_in - batch_size = x.shape[0] - past_seq_len = x.shape[1] - - if x_future_covariates is None: - raise_log( - ValueError("MambaEncoderDecoder requires future covariates."), - logger=logger, - ) - - future_seq_len = x_future_covariates.shape[1] - - # Build encoder input parts - encoder_parts = [] - - # Static covariates (expanded to past sequence length) - if self.static_cov_dim > 0 and x_static_covariates is not None: - if len(x_static_covariates.shape) > 2: - x_static_covariates = x_static_covariates.view(batch_size, -1) - static_past = x_static_covariates.unsqueeze(1).expand( - batch_size, past_seq_len, -1 - ) - encoder_parts.append(static_past) - - # Past target (optional) - if self.use_past_target: - x_past_target = x[:, :, : self.output_dim] - encoder_parts.append(x_past_target) - - # Past covariates - if self.past_cov_dim > 0: - x_past_cov = x[:, :, self.output_dim : self.output_dim + self.past_cov_dim] - encoder_parts.append(x_past_cov) - - encoder_input = torch.cat(encoder_parts, dim=2) - - # Build decoder input: [static | future_cov] - decoder_parts = [] - - if self.static_cov_dim > 0 and x_static_covariates is not None: - static_future = x_static_covariates.unsqueeze(1).expand( - batch_size, future_seq_len, -1 - ) - decoder_parts.append(static_future) - - decoder_parts.append(x_future_covariates) - decoder_input = torch.cat(decoder_parts, dim=2) - - # Encoder - encoder_embedded = self.dropout_layer( - self.activation(self.encoder_proj(encoder_input)) - ) - encoder_output = self.encoder_mamba(encoder_embedded) - - # Context token from encoder final timestep - context_token = encoder_output[:, -1:, :] - context_token = self.context_proj(context_token) - - # Decoder (prepend context token) - decoder_embedded = self.dropout_layer( - self.activation(self.decoder_proj(decoder_input)) - ) - decoder_input_with_context = torch.cat([context_token, decoder_embedded], dim=1) - decoder_output = self.decoder_mamba(decoder_input_with_context) - - # Take last output_chunk_length timesteps (exclude context token position) - last_steps = decoder_output[:, -self.output_chunk_length :, :] - - # Output projection - output = self.output_proj(last_steps) - output = output.view( - batch_size, self.output_chunk_length, self.output_dim, self.nr_params - ) - - return output diff --git a/st_forecast/custom_models/unified_temporal/__init__.py b/st_forecast/custom_models/unified_temporal/__init__.py deleted file mode 100644 index 9a00c03..0000000 --- a/st_forecast/custom_models/unified_temporal/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from st_forecast.custom_models.unified_temporal.model import UnifiedTemporalModel -from st_forecast.custom_models.unified_temporal.module import _UnifiedTemporalModule - -__all__ = ["UnifiedTemporalModel", "_UnifiedTemporalModule"] diff --git a/st_forecast/custom_models/unified_temporal/backends/__init__.py b/st_forecast/custom_models/unified_temporal/backends/__init__.py deleted file mode 100644 index f91aab5..0000000 --- a/st_forecast/custom_models/unified_temporal/backends/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from st_forecast.custom_models.unified_temporal.backends.base import TemporalBackend -from st_forecast.custom_models.unified_temporal.backends.lstm import LSTMBackend -from st_forecast.custom_models.unified_temporal.backends.mamba import MambaBackend -from st_forecast.custom_models.unified_temporal.backends.tcn import TCNBackend -from st_forecast.custom_models.unified_temporal.backends.attention import ( - AttentionBackend, -) - -__all__ = [ - "TemporalBackend", - "LSTMBackend", - "MambaBackend", - "TCNBackend", - "AttentionBackend", -] diff --git a/st_forecast/custom_models/unified_temporal/backends/attention.py b/st_forecast/custom_models/unified_temporal/backends/attention.py deleted file mode 100644 index ed6f331..0000000 --- a/st_forecast/custom_models/unified_temporal/backends/attention.py +++ /dev/null @@ -1,97 +0,0 @@ -import torch -import torch.nn as nn - - -def _generate_causal_mask(seq_len: int, device: torch.device) -> torch.Tensor: - mask = torch.triu(torch.ones(seq_len, seq_len, device=device), diagonal=1) - return mask.masked_fill(mask == 1, float("-inf")) - - -class AttentionBlock(nn.Module): - """Single transformer-style attention block with causal masking.""" - - def __init__( - self, - hidden_size: int, - num_heads: int, - dropout: float = 0.0, - ) -> None: - super().__init__() - self.attn = nn.MultiheadAttention( - embed_dim=hidden_size, - num_heads=num_heads, - dropout=dropout, - batch_first=True, - ) - self.norm1 = nn.LayerNorm(hidden_size) - self.norm2 = nn.LayerNorm(hidden_size) - - self.ffn = nn.Sequential( - nn.Linear(hidden_size, hidden_size * 4), - nn.SiLU(), - nn.Dropout(dropout), - nn.Linear(hidden_size * 4, hidden_size), - nn.Dropout(dropout), - ) - - def forward(self, x: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor: - # Self-attention with residual - attn_out, _ = self.attn(x, x, x, attn_mask=attn_mask) - x = self.norm1(x + attn_out) - - # FFN with residual - ffn_out = self.ffn(x) - x = self.norm2(x + ffn_out) - - return x - - -class AttentionBackend(nn.Module): - """Causal self-attention temporal backend for UnifiedTemporalModel. - - Uses multi-head attention with causal masking. - """ - - def __init__( - self, - hidden_size: int, - num_heads: int = 4, - num_layers: int = 2, - dropout: float = 0.0, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - self.num_heads = num_heads - self.num_layers = num_layers - self.dropout = dropout - - self.layers = nn.ModuleList( - [ - AttentionBlock( - hidden_size=hidden_size, - num_heads=num_heads, - dropout=dropout, - ) - for _ in range(num_layers) - ] - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Process sequence through causal self-attention. - - Parameters - ---------- - x : torch.Tensor - Input tensor of shape (B, L+T, hidden_size) - - Returns - ------- - torch.Tensor - Output tensor of shape (B, L+T, hidden_size) - """ - seq_len = x.size(1) - attn_mask = _generate_causal_mask(seq_len, x.device) - - for layer in self.layers: - x = layer(x, attn_mask) - return x diff --git a/st_forecast/custom_models/unified_temporal/backends/base.py b/st_forecast/custom_models/unified_temporal/backends/base.py deleted file mode 100644 index 3a0712d..0000000 --- a/st_forecast/custom_models/unified_temporal/backends/base.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Protocol - -import torch - - -class TemporalBackend(Protocol): - """Protocol for temporal backends in UnifiedTemporalModel.""" - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Process sequence through temporal model. - - Parameters - ---------- - x : torch.Tensor - Input tensor of shape (B, L+T, hidden_size) - - Returns - ------- - torch.Tensor - Output tensor of shape (B, L+T, hidden_size) - """ - ... diff --git a/st_forecast/custom_models/unified_temporal/backends/lstm.py b/st_forecast/custom_models/unified_temporal/backends/lstm.py deleted file mode 100644 index bc36bab..0000000 --- a/st_forecast/custom_models/unified_temporal/backends/lstm.py +++ /dev/null @@ -1,44 +0,0 @@ -import torch -import torch.nn as nn - - -class LSTMBackend(nn.Module): - """LSTM temporal backend for UnifiedTemporalModel. - - Ported from ExoLSTM architecture. - """ - - def __init__( - self, - hidden_size: int, - n_layers: int = 2, - dropout: float = 0.0, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - self.n_layers = n_layers - self.dropout = dropout - - self.lstm = nn.LSTM( - input_size=hidden_size, - hidden_size=hidden_size, - num_layers=n_layers, - dropout=dropout if n_layers > 1 else 0.0, - batch_first=True, - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Process sequence through LSTM. - - Parameters - ---------- - x : torch.Tensor - Input tensor of shape (B, L+T, hidden_size) - - Returns - ------- - torch.Tensor - Output tensor of shape (B, L+T, hidden_size) - """ - output, _ = self.lstm(x) - return output diff --git a/st_forecast/custom_models/unified_temporal/backends/mamba.py b/st_forecast/custom_models/unified_temporal/backends/mamba.py deleted file mode 100644 index 94f896e..0000000 --- a/st_forecast/custom_models/unified_temporal/backends/mamba.py +++ /dev/null @@ -1,52 +0,0 @@ -import torch -import torch.nn as nn - -try: - from mambapy.mamba import Mamba, MambaConfig -except ImportError: - raise ImportError( - "Mamba is not installed. Please install it using `pip install mambapy`." - ) - - -class MambaBackend(nn.Module): - """Mamba temporal backend for UnifiedTemporalModel. - - Ported from ExoMamba architecture. - """ - - def __init__( - self, - hidden_size: int, - n_layers: int = 2, - d_state: int = 16, - expand_factor: int = 2, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - self.n_layers = n_layers - self.d_state = d_state - self.expand_factor = expand_factor - - self.config = MambaConfig( - d_model=hidden_size, - n_layers=n_layers, - d_state=d_state, - expand_factor=expand_factor, - ) - self.mamba = Mamba(self.config) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Process sequence through Mamba. - - Parameters - ---------- - x : torch.Tensor - Input tensor of shape (B, L+T, hidden_size) - - Returns - ------- - torch.Tensor - Output tensor of shape (B, L+T, hidden_size) - """ - return self.mamba(x) diff --git a/st_forecast/custom_models/unified_temporal/backends/tcn.py b/st_forecast/custom_models/unified_temporal/backends/tcn.py deleted file mode 100644 index baa951c..0000000 --- a/st_forecast/custom_models/unified_temporal/backends/tcn.py +++ /dev/null @@ -1,108 +0,0 @@ -import torch -import torch.nn as nn - - -class TCNBlock(nn.Module): - """Single TCN block with dilated convolution and residual connection.""" - - def __init__( - self, - hidden_size: int, - kernel_size: int, - dilation: int, - dropout: float = 0.0, - ) -> None: - super().__init__() - padding = (kernel_size - 1) * dilation - - self.conv1 = nn.Conv1d( - hidden_size, - hidden_size, - kernel_size=kernel_size, - dilation=dilation, - padding=padding, - ) - self.conv2 = nn.Conv1d( - hidden_size, - hidden_size, - kernel_size=kernel_size, - dilation=dilation, - padding=padding, - ) - self.activation = nn.SiLU() - self.dropout = nn.Dropout(dropout) - self.norm = nn.LayerNorm(hidden_size) - self.padding = padding - - def forward(self, x: torch.Tensor) -> torch.Tensor: - # x: (B, hidden_size, L) - residual = x - - out = self.conv1(x) - out = out[:, :, : -self.padding] if self.padding > 0 else out - out = self.activation(out) - out = self.dropout(out) - - out = self.conv2(out) - out = out[:, :, : -self.padding] if self.padding > 0 else out - - out = out + residual - # LayerNorm expects (B, L, C), so transpose - out = out.transpose(1, 2) - out = self.norm(out) - out = out.transpose(1, 2) - - return out - - -class TCNBackend(nn.Module): - """TCN temporal backend for UnifiedTemporalModel. - - Uses dilated 1D convolutions with residual connections. - """ - - def __init__( - self, - hidden_size: int, - kernel_sizes: list[int] | None = None, - dropout: float = 0.0, - ) -> None: - super().__init__() - self.hidden_size = hidden_size - self.kernel_sizes = kernel_sizes or [15, 5, 3] - self.dropout = dropout - - blocks = [] - for i, kernel_size in enumerate(self.kernel_sizes): - dilation = 2**i - blocks.append( - TCNBlock( - hidden_size=hidden_size, - kernel_size=kernel_size, - dilation=dilation, - dropout=dropout, - ) - ) - self.blocks = nn.ModuleList(blocks) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Process sequence through TCN. - - Parameters - ---------- - x : torch.Tensor - Input tensor of shape (B, L+T, hidden_size) - - Returns - ------- - torch.Tensor - Output tensor of shape (B, L+T, hidden_size) - """ - # Conv1d expects (B, C, L), input is (B, L, C) - out = x.transpose(1, 2) - - for block in self.blocks: - out = block(out) - - # Back to (B, L, C) - return out.transpose(1, 2) diff --git a/st_forecast/custom_models/unified_temporal/feature_utils.py b/st_forecast/custom_models/unified_temporal/feature_utils.py deleted file mode 100644 index 69657a7..0000000 --- a/st_forecast/custom_models/unified_temporal/feature_utils.py +++ /dev/null @@ -1,125 +0,0 @@ -import torch - - -def extend_past_to_future_shape( - past_features: torch.Tensor, - future_len: int, -) -> tuple[torch.Tensor, torch.Tensor]: - """Extend past features into future timesteps with zeros and create a flag tensor. - - Parameters - ---------- - past_features - Tensor of shape (B, L, P) containing past covariate features - future_len - Number of future timesteps (T) - - Returns - ------- - tuple[torch.Tensor, torch.Tensor] - extended_past: (B, L+T, P) - zeros padded for future timesteps - flag_tensor: (B, L+T, 1) - 0=observed (past), 1=missing (future) - """ - batch_size, past_len, num_features = past_features.shape - total_len = past_len + future_len - - # Create extended tensor with zeros for future - extended_past = torch.zeros( - batch_size, - total_len, - num_features, - dtype=past_features.dtype, - device=past_features.device, - ) - extended_past[:, :past_len, :] = past_features - - # Create flag tensor (0=observed, 1=missing) - flag_tensor = torch.zeros( - batch_size, - total_len, - 1, - dtype=past_features.dtype, - device=past_features.device, - ) - flag_tensor[:, past_len:, :] = 1.0 - - return extended_past, flag_tensor - - -def concatenate_features( - static_features: torch.Tensor | None, - past_extended: torch.Tensor | None, - flag_tensor: torch.Tensor | None, - future_padded: torch.Tensor, - use_past_covariates: bool, -) -> torch.Tensor: - """Concatenate all feature tensors along the feature dimension. - - The concatenation order is: [static | past_extended | flag | future_padded] - If use_past_covariates=False, only [static | future_padded] is concatenated. - - Parameters - ---------- - static_features - Static features expanded to (B, L+T, S) or None - past_extended - Extended past features (B, L+T, P) or None (only used if use_past_covariates=True) - flag_tensor - Flag tensor (B, L+T, 1) or None (only used if use_past_covariates=True) - future_padded - Future features padded to (B, L+T, F) - use_past_covariates - Whether to include past covariates and flag in concatenation - - Returns - ------- - torch.Tensor - Concatenated features of shape (B, L+T, total_features) - """ - features_to_concat = [] - - if static_features is not None: - features_to_concat.append(static_features) - - if use_past_covariates: - if past_extended is not None: - features_to_concat.append(past_extended) - if flag_tensor is not None: - features_to_concat.append(flag_tensor) - - features_to_concat.append(future_padded) - - return torch.cat(features_to_concat, dim=2) - - -def pad_future_to_full_length( - future_features: torch.Tensor, - past_len: int, -) -> torch.Tensor: - """Pad future features with zeros at the beginning to match full sequence length. - - Parameters - ---------- - future_features - Tensor of shape (B, T, F) containing future covariate features - past_len - Number of past timesteps (L) - - Returns - ------- - torch.Tensor - Padded tensor of shape (B, L+T, F) with zeros for past timesteps - """ - batch_size, future_len, num_features = future_features.shape - total_len = past_len + future_len - - padded = torch.zeros( - batch_size, - total_len, - num_features, - dtype=future_features.dtype, - device=future_features.device, - ) - padded[:, past_len:, :] = future_features - - return padded diff --git a/st_forecast/custom_models/unified_temporal/model.py b/st_forecast/custom_models/unified_temporal/model.py deleted file mode 100644 index a2fb7bb..0000000 --- a/st_forecast/custom_models/unified_temporal/model.py +++ /dev/null @@ -1,217 +0,0 @@ -"""UnifiedTemporalModel - Flexible temporal forecasting with multiple backends.""" - -import torch - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.torch_forecasting_model import MixedCovariatesTorchModel - -from st_forecast.custom_models.unified_temporal.module import _UnifiedTemporalModule - -MixedCovariatesTrainTensorType = tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor -] - -logger = get_logger(__name__) - - -class UnifiedTemporalModel(MixedCovariatesTorchModel): - def __init__( - self, - input_chunk_length: int, - output_chunk_length: int, - output_chunk_shift: int = 0, - hidden_size: int = 64, - temporal_model: str = "lstm", - use_past_covariates: bool = True, - use_static_covariates: bool = True, - use_input_linear: bool = True, - dropout: float = 0.1, - # LSTM-specific - n_lstm_layers: int = 2, - # Mamba-specific - n_mamba_layers: int = 2, - d_state: int = 16, - expand_factor: int = 2, - # TCN-specific - kernel_sizes: list[int] | None = None, - # Attention-specific - num_heads: int = 4, - num_attention_layers: int = 2, - **kwargs, - ): - """Unified temporal forecasting model with multiple backend options. - - This model supports multiple temporal backends (LSTM, Mamba, TCN, Attention) - with a unified feature handling approach. When use_past_covariates=True, it - extends past features into future timesteps with a flag tensor indicating - whether data is observed or missing. Missing values are filled with learnable - imputation parameters. - - Parameters - ---------- - input_chunk_length - Number of time steps in the past to take as a model input (per chunk). - output_chunk_length - Number of time steps predicted at once (per chunk) by the internal model. - output_chunk_shift - Optionally, the number of steps to shift the start of the output chunk - into the future (relative to the input chunk end). - hidden_size - The size of the hidden layers in the model. - temporal_model - Which temporal backend to use. One of: "lstm", "mamba", "tcn", "attention". - use_past_covariates - If True, extends past covariates into future with flag tensor. - If False, behaves like ExoLSTM (only uses future covariates). - use_static_covariates - Whether the model should use static covariates. - use_input_linear - If True, applies a linear projection before the temporal backend. - If False, feeds concatenated features directly to the backend. - dropout - The dropout probability to be used in the model. - n_lstm_layers - Number of LSTM layers (only used when temporal_model="lstm"). - n_mamba_layers - Number of Mamba layers (only used when temporal_model="mamba"). - d_state - State dimension for Mamba blocks (only used when temporal_model="mamba"). - expand_factor - Expansion factor for Mamba (only used when temporal_model="mamba"). - kernel_sizes - Kernel sizes for TCN blocks (only used when temporal_model="tcn"). - Defaults to [15, 5, 3]. - num_heads - Number of attention heads (only used when temporal_model="attention"). - num_attention_layers - Number of attention layers (only used when temporal_model="attention"). - **kwargs - Optional arguments to initialize the pytorch_lightning.Module, - pytorch_lightning.Trainer, and Darts' TorchForecastingModel. - """ - super().__init__(**self._extract_torch_model_params(**self.model_params)) - - # Extract pytorch lightning module kwargs - self.pl_module_params = self._extract_pl_module_params(**self.model_params) - - # Store model configuration - self.hidden_size = hidden_size - self.temporal_model = temporal_model - self.use_past_covariates = use_past_covariates - self.use_input_linear = use_input_linear - self.dropout = dropout - - # Backend-specific params - self.n_lstm_layers = n_lstm_layers - self.n_mamba_layers = n_mamba_layers - self.d_state = d_state - self.expand_factor = expand_factor - self.kernel_sizes = kernel_sizes - self.num_heads = num_heads - self.num_attention_layers = num_attention_layers - - self._considers_static_covariates = use_static_covariates - - def _create_model( - self, train_sample: MixedCovariatesTrainTensorType - ) -> torch.nn.Module: - ( - past_target, - past_covariates, - historic_future_covariates, - future_covariates, - static_covariates, - future_target, - ) = train_sample - - # Target dimension - output_dim = future_target.shape[1] - - # Validate covariates - if future_covariates is None: - raise_log( - ValueError("UnifiedTemporalModel requires future covariates."), - logger=logger, - ) - - future_cov_dim = future_covariates.shape[1] - if future_cov_dim == 0: - raise_log( - ValueError( - "UnifiedTemporalModel requires non-empty future covariates." - ), - logger=logger, - ) - - # Past covariate dimension (can be 0 if use_past_covariates=False) - if self.use_past_covariates: - if past_covariates is None: - raise_log( - ValueError( - "UnifiedTemporalModel with use_past_covariates=True " - "requires past covariates." - ), - logger=logger, - ) - past_cov_dim = past_covariates.shape[1] - else: - past_cov_dim = 0 - - # Input dimension for the module - input_dim = future_cov_dim if past_cov_dim == 0 else past_cov_dim - - # Handle static covariates - if static_covariates is not None: - if len(static_covariates.shape) > 2: - static_cov_dim = ( - static_covariates.shape[0] * static_covariates.shape[-1] - ) - logger.warning( - f"Static covariates have unexpected shape {static_covariates.shape}. " - f"Flattening to dimension {static_cov_dim}." - ) - else: - static_cov_dim = static_covariates.shape[-1] - else: - static_cov_dim = 0 - - # Number of parameters (1 for point forecasts, more for probabilistic) - nr_params = 1 if self.likelihood is None else self.likelihood.num_parameters - - # Create module - model = _UnifiedTemporalModule( - input_dim=input_dim, - output_dim=output_dim, - nr_params=nr_params, - future_cov_dim=future_cov_dim, - past_cov_dim=past_cov_dim, - static_cov_dim=static_cov_dim, - hidden_size=self.hidden_size, - temporal_model=self.temporal_model, - use_past_covariates=self.use_past_covariates, - use_input_linear=self.use_input_linear, - dropout=self.dropout, - n_lstm_layers=self.n_lstm_layers, - n_mamba_layers=self.n_mamba_layers, - d_state=self.d_state, - expand_factor=self.expand_factor, - kernel_sizes=self.kernel_sizes, - num_heads=self.num_heads, - num_attention_layers=self.num_attention_layers, - **self.pl_module_params, - ) - - return model - - @property - def supports_static_covariates(self) -> bool: - return True - - @property - def supports_multivariate(self) -> bool: - return True - - @property - def ignores_past_target_values(self) -> bool: - """Indicates that this model does not use target's past values for prediction.""" - return True diff --git a/st_forecast/custom_models/unified_temporal/module.py b/st_forecast/custom_models/unified_temporal/module.py deleted file mode 100644 index 5c4c727..0000000 --- a/st_forecast/custom_models/unified_temporal/module.py +++ /dev/null @@ -1,297 +0,0 @@ -"""UnifiedTemporalModule - PLForecastingModule for UnifiedTemporalModel.""" - -from typing import Optional - -import torch -import torch.nn as nn - -from darts.logging import get_logger, raise_log -from darts.models.forecasting.pl_forecasting_module import ( - PLForecastingModule, - io_processor, -) -from darts.utils.torch import MonteCarloDropout - -from st_forecast.custom_models.unified_temporal.feature_utils import ( - concatenate_features, - pad_future_to_full_length, -) - -logger = get_logger(__name__) - - -class _UnifiedTemporalModule(PLForecastingModule): - def __init__( - self, - input_dim: int, - output_dim: int, - nr_params: int, - future_cov_dim: int, - past_cov_dim: int, - static_cov_dim: int, - hidden_size: int, - temporal_model: str, - use_past_covariates: bool, - dropout: float, - # New options - use_input_linear: bool = True, - # LSTM-specific - n_lstm_layers: int = 2, - # Mamba-specific - n_mamba_layers: int = 2, - d_state: int = 16, - expand_factor: int = 2, - # TCN-specific - kernel_sizes: list[int] | None = None, - # Attention-specific - num_heads: int = 4, - num_attention_layers: int = 2, - **kwargs, - ) -> None: - super().__init__(**kwargs) - - self.input_dim = input_dim - self.output_dim = output_dim - self.nr_params = nr_params - self.future_cov_dim = future_cov_dim - self.past_cov_dim = past_cov_dim - self.static_cov_dim = static_cov_dim - self.hidden_size = hidden_size - self.temporal_model = temporal_model - self.use_past_covariates = use_past_covariates - self.use_input_linear = use_input_linear - self.dropout = dropout - - # Compute combined feature dimension - # If use_past_covariates: [static | past_extended | flag | future_padded] -> S + P + 1 + F - # If not: [static | future_padded] -> S + F - if use_past_covariates: - combined_dim = static_cov_dim + past_cov_dim + 1 + future_cov_dim - else: - combined_dim = static_cov_dim + future_cov_dim - - self.combined_dim = combined_dim - - # Learnable imputation values for missing past covariates in future timesteps - if use_past_covariates and past_cov_dim > 0: - self.missing_past_cov_impute = nn.Parameter(torch.zeros(past_cov_dim)) - else: - self.register_parameter("missing_past_cov_impute", None) - - # Input embedding layer (optional) - if use_input_linear: - self.input_linear = nn.Linear(combined_dim, hidden_size) - backend_input_size = hidden_size - else: - self.input_linear = None - backend_input_size = combined_dim - - # Activation and dropout - self.activation = nn.SiLU() - self.dropout_layer = MonteCarloDropout(dropout) - - # Temporal backend - self.backend = self._create_backend( - temporal_model=temporal_model, - hidden_size=backend_input_size, - dropout=dropout, - n_lstm_layers=n_lstm_layers, - n_mamba_layers=n_mamba_layers, - d_state=d_state, - expand_factor=expand_factor, - kernel_sizes=kernel_sizes, - num_heads=num_heads, - num_attention_layers=num_attention_layers, - ) - - # Output projection - self.output_proj = nn.Linear(backend_input_size, output_dim * nr_params) - - def _create_backend( - self, - temporal_model: str, - hidden_size: int, - dropout: float, - n_lstm_layers: int, - n_mamba_layers: int, - d_state: int, - expand_factor: int, - kernel_sizes: list[int] | None, - num_heads: int, - num_attention_layers: int, - ) -> nn.Module: - match temporal_model: - case "lstm": - from st_forecast.custom_models.unified_temporal.backends import ( - LSTMBackend, - ) - - return LSTMBackend(hidden_size, n_lstm_layers, dropout) - case "mamba": - from st_forecast.custom_models.unified_temporal.backends import ( - MambaBackend, - ) - - return MambaBackend(hidden_size, n_mamba_layers, d_state, expand_factor) - case "tcn": - from st_forecast.custom_models.unified_temporal.backends import ( - TCNBackend, - ) - - return TCNBackend(hidden_size, kernel_sizes or [3, 5, 7], dropout) - case "attention": - from st_forecast.custom_models.unified_temporal.backends import ( - AttentionBackend, - ) - - return AttentionBackend( - hidden_size, num_heads, num_attention_layers, dropout - ) - case _: - raise_log( - ValueError( - f"Unknown temporal_model: {temporal_model}. " - f"Expected one of: lstm, mamba, tcn, attention" - ), - logger=logger, - ) - - def _extend_past_with_learnable_impute( - self, - past_features: torch.Tensor, - future_len: int, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Extend past features into future timesteps with learnable imputation. - - Parameters - ---------- - past_features - Tensor of shape (B, L, P) containing past covariate features - future_len - Number of future timesteps (T) - - Returns - ------- - tuple[torch.Tensor, torch.Tensor] - extended_past: (B, L+T, P) - learnable values for future timesteps - flag_tensor: (B, L+T, 1) - 0=observed (past), 1=missing (future) - """ - batch_size, past_len, num_features = past_features.shape - total_len = past_len + future_len - - # Create extended tensor - extended_past = torch.zeros( - batch_size, - total_len, - num_features, - dtype=past_features.dtype, - device=past_features.device, - ) - # Copy past values - extended_past[:, :past_len, :] = past_features - # Fill future with learnable imputation values (broadcast across batch and time) - extended_past[:, past_len:, :] = self.missing_past_cov_impute.unsqueeze( - 0 - ).unsqueeze(0) - - # Create flag tensor (0=observed, 1=missing) - flag_tensor = torch.zeros( - batch_size, - total_len, - 1, - dtype=past_features.dtype, - device=past_features.device, - ) - flag_tensor[:, past_len:, :] = 1.0 - - return extended_past, flag_tensor - - @io_processor - def forward( - self, x_in: tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]] - ) -> torch.Tensor: - x, x_future_covariates, x_static_covariates = x_in - - batch_size = x.shape[0] - past_len = x.shape[1] - - # Validate future covariates - if x_future_covariates is None: - raise_log( - ValueError("UnifiedTemporalModule requires future covariates."), - logger=logger, - ) - - future_len = x_future_covariates.shape[1] - total_len = past_len + future_len - - # Process static covariates - static_expanded: torch.Tensor | None = None - if self.static_cov_dim > 0 and x_static_covariates is not None: - if len(x_static_covariates.shape) > 2: - x_static_covariates = x_static_covariates.view(batch_size, -1) - static_expanded = x_static_covariates.unsqueeze(1).expand( - batch_size, total_len, -1 - ) - - # Process features based on use_past_covariates flag - if self.use_past_covariates: - # Extract past covariates from x (excluding autoregressive target features) - # x contains [target | past_covariates], extract past_covariates portion - x_past_covariates = x[ - :, :, self.output_dim : self.output_dim + self.past_cov_dim - ] - - # Extend past covariates with learnable imputation for future timesteps - past_extended, flag_tensor = self._extend_past_with_learnable_impute( - x_past_covariates, future_len - ) - - # Pad future covariates to full length - future_padded = pad_future_to_full_length(x_future_covariates, past_len) - - # Concatenate: [static | past_extended | flag | future_padded] - combined = concatenate_features( - static_features=static_expanded, - past_extended=past_extended, - flag_tensor=flag_tensor, - future_padded=future_padded, - use_past_covariates=True, - ) - else: - # ExoLSTM-equivalent: only future covariates + static - future_padded = pad_future_to_full_length(x_future_covariates, past_len) - - # Concatenate: [static | future_padded] - combined = concatenate_features( - static_features=static_expanded, - past_extended=None, - flag_tensor=None, - future_padded=future_padded, - use_past_covariates=False, - ) - - # Input embedding (optional) - if self.input_linear is not None: - embedded = self.input_linear(combined) - embedded = self.activation(embedded) - embedded = self.dropout_layer(embedded) - else: - # Direct input to backend without linear projection - embedded = combined - - # Temporal processing - temporal_output = self.backend(embedded) - - # Extract last T timesteps (output_chunk_length) - last_steps = temporal_output[:, -self.output_chunk_length :, :] - - # Output projection - output = self.output_proj(last_steps) - - # Reshape to (batch_size, output_chunk_length, output_dim, nr_params) - output = output.view( - batch_size, self.output_chunk_length, self.output_dim, self.nr_params - ) - - return output diff --git a/st_forecast/model_helper.py b/st_forecast/model_helper.py index 0308d4f..9dd2546 100644 --- a/st_forecast/model_helper.py +++ b/st_forecast/model_helper.py @@ -11,9 +11,7 @@ import random from st_forecast.custom_models.FANMixer import FANMixerModel -from st_forecast.custom_models.ExoMamba import ExoMambaModel from st_forecast.custom_models.ExoLSTM import ExoLSTMModel -from st_forecast.custom_models.unified_temporal import UnifiedTemporalModel from darts.utils.likelihood_models import QuantileRegression @@ -29,6 +27,21 @@ torch.serialization.add_safe_globals([QuantileRegression]) +_ENCODER_MAP: dict[str, dict | None] = { + "week": {"cyclic": {"future": ["week"]}}, + "month": {"cyclic": {"future": ["month"]}}, + "none": None, +} + + +def _resolve_encoder(encoder: str) -> dict | None: + if encoder not in _ENCODER_MAP: + raise ValueError( + f"Unknown encoder: {encoder!r}. Supported: {list(_ENCODER_MAP)}" + ) + return _ENCODER_MAP[encoder] + + # Set the random seed for reproducibility random.seed(42) torch.manual_seed(42) @@ -113,7 +126,7 @@ def initialize_model(config: RunConfig) -> tuple[object, TrainingLossLogger]: "n_epochs": config.n_epochs, "save_checkpoints": True, "work_dir": config.work_dir, - "add_encoders": {"cyclic": {"future": ["week"]}}, + "add_encoders": _resolve_encoder(config.encoder), "log_tensorboard": True, "model_name": config.model_name, "force_reset": True, @@ -144,28 +157,8 @@ def _get_model_class(model_type: str): return TiDEModel case "FANMixer": return FANMixerModel - case "ExoMamba": - return ExoMambaModel case "ExoLSTM": return ExoLSTMModel - case "UnifiedTemporal": - return UnifiedTemporalModel - case "LSTMEncDec": - from st_forecast.custom_models.lstm_enc_dec import LSTMEncoderDecoderModel - - return LSTMEncoderDecoderModel - case "MambaEncDec": - from st_forecast.custom_models.mamba_enc_dec import MambaEncoderDecoderModel - - return MambaEncoderDecoderModel - case "LSTMEncMLP": - from st_forecast.custom_models.lstm_enc_mlp import LSTMEncMLPModel - - return LSTMEncMLPModel - case "LSTM": - from st_forecast.custom_models.lstm import LSTMModel - - return LSTMModel case "RNN": from darts.models import RNNModel @@ -173,7 +166,7 @@ def _get_model_class(model_type: str): case _: raise ValueError( f"Unknown model_type: {model_type}. " - "Supported: TFT, TSMixer, TiDE, FANMixer, ExoMamba, ExoLSTM, UnifiedTemporal, LSTMEncDec, MambaEncDec, LSTMEncMLP, LSTM" + "Supported: TFT, TSMixer, TiDE, FANMixer, ExoLSTM, RNN" ) @@ -221,17 +214,6 @@ def _get_model_specific_kwargs(config: RunConfig) -> dict: "hidden_size": config.hidden_size, "dropout": config.dropout, "n_lstm_layers": extra.get("n_lstm_layers", 2), - "use_reversible_instance_norm": False, - } - case "ExoMamba": - return { - "hidden_size": config.hidden_size, - "dropout": config.dropout, - "n_mamba_layers": extra.get("n_mamba_layers", 2), - "d_state": extra.get("d_state", 16), - "expand_factor": extra.get("expand_factor", 2), - "use_reversible_instance_norm": False, - "log_tensorboard": False, } case "TiDE": return { @@ -245,57 +227,6 @@ def _get_model_specific_kwargs(config: RunConfig) -> dict: "temporal_decoder_hidden": config.temporal_decoder_hidden_size, "use_reversible_instance_norm": use_rin, } - case "UnifiedTemporal": - return { - "hidden_size": config.hidden_size, - "dropout": config.dropout, - "temporal_model": extra.get("temporal_model", "lstm"), - "use_past_covariates": extra.get("use_past_covariates", True), - "use_input_linear": extra.get("use_input_linear", True), - "n_lstm_layers": extra.get("n_lstm_layers", 2), - "n_mamba_layers": extra.get("n_mamba_layers", 2), - "d_state": extra.get("d_state", 16), - "expand_factor": extra.get("expand_factor", 2), - "kernel_sizes": extra.get("kernel_sizes", [15, 5, 3]), - "num_heads": extra.get("num_heads", 4), - "num_attention_layers": extra.get("num_attention_layers", 2), - } - case "LSTMEncDec": - use_past_target = extra.get("use_past_target", True) - return { - "hidden_size": config.hidden_size, - "n_layers": extra.get("n_lstm_layers", 2), - "dropout": config.dropout, - "use_past_target": use_past_target, - "use_learned_init": extra.get("use_learned_init", True), - "use_skip_connection": extra.get("use_skip_connection", True), - "use_static_covariates": extra.get("use_static_covariates", True), - "use_reversible_instance_norm": use_rin and use_past_target, - } - case "MambaEncDec": - use_past_target = extra.get("use_past_target", True) - return { - "hidden_size": config.hidden_size, - "n_layers": extra.get("n_mamba_layers", 2), - "d_state": extra.get("d_state", 16), - "expand_factor": extra.get("expand_factor", 2), - "dropout": config.dropout, - "use_past_target": use_past_target, - "use_static_covariates": extra.get("use_static_covariates", True), - "use_reversible_instance_norm": use_rin and use_past_target, - } - case "LSTM": - use_past_target = extra.get("use_past_target", True) - return { - "hidden_size": config.hidden_size, - "n_lstm_layers": extra.get("n_lstm_layers", 2), - "dropout": config.dropout, - "use_past_target": use_past_target, - "use_learned_init": extra.get("use_learned_init", True), - "use_static_covariates": extra.get("use_static_covariates", True), - "use_reversible_instance_norm": use_rin and use_past_target, - "obs_mask_prob": extra.get("obs_mask_prob", 0.0), - } case "RNN": return { "model": extra.get("rnn_model", "LSTM"), @@ -307,20 +238,6 @@ def _get_model_specific_kwargs(config: RunConfig) -> dict: config.input_chunk_length + config.forecast_horizon, ), } - case "LSTMEncMLP": - use_past_target = extra.get("use_past_target", True) - return { - "hidden_size": config.hidden_size, - "n_layers": extra.get("n_lstm_layers", 2), - "dropout": config.dropout, - "use_past_target": use_past_target, - "use_learned_init": extra.get("use_learned_init", True), - "num_decoder_blocks": extra.get("num_decoder_blocks", 2), - "decoder_activation": extra.get("decoder_activation", "ReLU"), - "use_layer_norm": extra.get("use_layer_norm", False), - "use_static_covariates": extra.get("use_static_covariates", True), - "use_reversible_instance_norm": use_rin and use_past_target, - } case _: return {} @@ -616,7 +533,6 @@ def initialize_ExoLSTM(config_model): hidden_size=hidden_size, dropout=dropout, n_lstm_layers=n_lstm_layers, - use_reversible_instance_norm=False, optimizer_cls=optimizer_cls, optimizer_kwargs=optimizer_kwargs, pl_trainer_kwargs=pl_trainer_kwargs, @@ -637,93 +553,6 @@ def initialize_ExoLSTM(config_model): return model, loss_logger -def initialize_ExoMamba(config_model): - work_dir = config_model["work_dir"] - model_name = config_model["model_name"] - - # Hyperparameters - n_epochs = config_model["n_epochs"] - weight_decay = config_model["weight_decay"] - lr = config_model["lr"] - hidden_size = config_model["hidden_size"] - dropout = config_model["dropout"] - n_mamba_layers = config_model["n_mamba_layers"] - d_state = config_model["d_state"] - expand_factor = config_model["expand_factor"] - - batch_size = config_model["batch_size"] - input_chunk_length = config_model["input_chunk_length"] - forecast_horizon = config_model["forecast_horizon"] - - # Scheduler - factor = config_model["factor"] - patience = config_model["patience"] - - lr_scheduler_cls = torch.optim.lr_scheduler.ReduceLROnPlateau - lr_scheduler_kwargs = { - "mode": "min", - "factor": factor, - "patience": patience, - } - - # Early stopping - early_stopping_patience = config_model["early_stopping_patience"] - loss_logger = LossLogger() - early_stopper = EarlyStopping( - "val_loss", min_delta=0.001, patience=early_stopping_patience, verbose=True - ) - callbacks = [loss_logger, early_stopper] - - # Loss function and Optimizer - quantiles = [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95] - optimizer_cls = torch.optim.Adam - optimizer_kwargs = {"lr": lr, "weight_decay": weight_decay} - - # Lightning Trainer - pl_trainer_kwargs = { - "max_epochs": n_epochs, - "accelerator": "cpu", - "callbacks": callbacks, - "gradient_clip_val": 0.1, - "enable_progress_bar": True, - "log_every_n_steps": 1, - "enable_checkpointing": True, - "check_val_every_n_epoch": 1, - "log_every_n_steps": 1, - "enable_progress_bar": True, - "logger": True, - "deterministic": True, - } - - model = ExoMambaModel( - input_chunk_length=input_chunk_length, - output_chunk_length=forecast_horizon, - hidden_size=hidden_size, - dropout=dropout, - n_mamba_layers=n_mamba_layers, - d_state=d_state, - expand_factor=expand_factor, - batch_size=int(batch_size), - random_state=42, - n_epochs=n_epochs, - save_checkpoints=True, - work_dir=work_dir, - add_encoders={"cyclic": {"future": ["month"]}}, - log_tensorboard=False, - model_name=model_name, - force_reset=True, - optimizer_cls=optimizer_cls, - optimizer_kwargs=optimizer_kwargs, - pl_trainer_kwargs=pl_trainer_kwargs, - likelihood=QuantileRegression(quantiles=quantiles), - lr_scheduler_cls=lr_scheduler_cls, - lr_scheduler_kwargs=lr_scheduler_kwargs, - use_reversible_instance_norm=False, - ) - - return model, loss_logger - - def initialize_TiDE(config_model): work_dir = config_model["work_dir"] model_name = config_model["model_name"] diff --git a/st_forecast/pipeline/artifacts.py b/st_forecast/pipeline/artifacts.py index 7545821..fd69e98 100644 --- a/st_forecast/pipeline/artifacts.py +++ b/st_forecast/pipeline/artifacts.py @@ -12,13 +12,7 @@ "tsmixer": "TSMixer", "tide": "TiDE", "fanmixer": "FANMixer", - "exomamba": "ExoMamba", "exolstm": "ExoLSTM", - "unifiedtemporal": "UnifiedTemporal", - "lstmencdec": "LSTMEncDec", - "mambaencdec": "MambaEncDec", - "lstmencmlp": "LSTMEncMLP", - "lstm": "LSTM", "rnn": "RNN", "persistence": "Persistence", } @@ -121,6 +115,7 @@ class RunConfig: accelerator: str = "auto" checkpoint_strategy: str = "best" # "best" | "last" optimizer: str = "adam" # "adam" | "adamw" | "sgd" + encoder: str = "month" # "week" | "month" | "none" # Work directory work_dir: str | None = None @@ -264,6 +259,7 @@ def to_model_config_dict(self) -> dict: "checkpoint_strategy": self.checkpoint_strategy, "optimizer": self.optimizer, "quantiles": self.quantiles, + "encoder": self.encoder, } base.update(self.extra_hyperparams) return base diff --git a/st_forecast/pipeline/inference.py b/st_forecast/pipeline/inference.py index d0870d6..c02ac1c 100644 --- a/st_forecast/pipeline/inference.py +++ b/st_forecast/pipeline/inference.py @@ -67,22 +67,6 @@ def load_model(config: RunConfig, run_folder: Path) -> object: from st_forecast.custom_models.ExoLSTM import ExoLSTMModel return ExoLSTMModel.load(str(model_path)) - elif model_type == "EXOMAMBA": - from st_forecast.custom_models.ExoMamba import ExoMambaModel - - return ExoMambaModel.load(str(model_path)) - elif model_type == "LSTMENCDEC": - from st_forecast.custom_models.lstm_enc_dec import LSTMEncoderDecoderModel - - return LSTMEncoderDecoderModel.load(str(model_path)) - elif model_type == "MAMBAENCDEC": - from st_forecast.custom_models.mamba_enc_dec import MambaEncoderDecoderModel - - return MambaEncoderDecoderModel.load(str(model_path)) - elif model_type == "LSTM": - from st_forecast.custom_models.lstm import LSTMModel - - return LSTMModel.load(str(model_path)) elif model_type == "RNN": from darts.models import RNNModel diff --git a/st_forecast/pipeline/training.py b/st_forecast/pipeline/training.py index 1ddd011..e01bd27 100644 --- a/st_forecast/pipeline/training.py +++ b/st_forecast/pipeline/training.py @@ -7,12 +7,7 @@ from st_forecast import model_helper from st_forecast.custom_models.ExoLSTM import ExoLSTMModel -from st_forecast.custom_models.ExoMamba import ExoMambaModel from st_forecast.custom_models.FANMixer import FANMixerModel -from st_forecast.custom_models.lstm_enc_dec import LSTMEncoderDecoderModel -from st_forecast.custom_models.lstm import LSTMModel -from st_forecast.custom_models.mamba_enc_dec import MambaEncoderDecoderModel -from st_forecast.custom_models.unified_temporal import UnifiedTemporalModel from st_forecast.training_components import LossLogger if TYPE_CHECKING: @@ -51,18 +46,8 @@ def _get_model_class(model_type: str): return TiDEModel case "FANMixer": return FANMixerModel - case "ExoMamba": - return ExoMambaModel case "ExoLSTM": return ExoLSTMModel - case "UnifiedTemporal": - return UnifiedTemporalModel - case "LSTMEncDec": - return LSTMEncoderDecoderModel - case "MambaEncDec": - return MambaEncoderDecoderModel - case "LSTM": - return LSTMModel case "RNN": from darts.models import RNNModel @@ -70,7 +55,7 @@ def _get_model_class(model_type: str): case _: raise ValueError( f"Unknown model_type: {model_type}. " - "Supported: TFT, TSMixer, TiDE, FANMixer, ExoMamba, ExoLSTM, UnifiedTemporal, LSTMEncDec, MambaEncDec, LSTM" + "Supported: TFT, TSMixer, TiDE, FANMixer, ExoLSTM, RNN" ) diff --git a/st_forecast/train_model.py b/st_forecast/train_model.py index 36a4821..2c3bd0a 100644 --- a/st_forecast/train_model.py +++ b/st_forecast/train_model.py @@ -54,7 +54,6 @@ prepare_validation_data, ) from st_forecast.custom_models.FANMixer import FANMixerModel -from st_forecast.custom_models.ExoMamba import ExoMambaModel from st_forecast.custom_models.ExoLSTM import ExoLSTMModel @@ -338,9 +337,6 @@ def drop_code_static_covariates(ts): elif model_type == "FANMixer": model, loss_logger = model_helper.initialize_FANMixer(config_model) - elif model_type == "ExoMamba": - model, loss_logger = model_helper.initialize_ExoMamba(config_model) - elif model_type == "ExoLSTM": model, loss_logger = model_helper.initialize_ExoLSTM(config_model) else: @@ -387,10 +383,6 @@ def drop_code_static_covariates(ts): best_model = FANMixerModel.load_from_checkpoint( model_name, work_dir=work_dir, best=True ) - elif model_type == "ExoMamba": - best_model = ExoMambaModel.load_from_checkpoint( - model_name, work_dir=work_dir, best=True - ) elif model_type == "ExoLSTM": best_model = ExoLSTMModel.load_from_checkpoint( model_name, work_dir=work_dir, best=True diff --git a/tests/custom_models/test_lstm_enc_dec.py b/tests/custom_models/test_lstm_enc_dec.py deleted file mode 100644 index 6ce9ada..0000000 --- a/tests/custom_models/test_lstm_enc_dec.py +++ /dev/null @@ -1,594 +0,0 @@ -"""Tests for LSTM Encoder-Decoder model.""" - -import numpy as np -import pandas as pd -import pytest -from darts import TimeSeries - -from st_forecast.custom_models.lstm_enc_dec import LSTMEncoderDecoderModel - - -@pytest.fixture -def synthetic_target() -> TimeSeries: - """Create synthetic target time series.""" - np.random.seed(42) - return TimeSeries.from_values(np.random.randn(100, 1).astype(np.float32)) - - -@pytest.fixture -def synthetic_past_cov() -> TimeSeries: - """Create synthetic past covariates.""" - np.random.seed(43) - return TimeSeries.from_values(np.random.randn(100, 3).astype(np.float32)) - - -@pytest.fixture -def synthetic_future_cov() -> TimeSeries: - """Create synthetic future covariates.""" - np.random.seed(44) - return TimeSeries.from_values(np.random.randn(120, 2).astype(np.float32)) - - -@pytest.fixture -def synthetic_target_with_static() -> TimeSeries: - """Create synthetic target with static covariates.""" - np.random.seed(42) - target = TimeSeries.from_values(np.random.randn(100, 1).astype(np.float32)) - return target.with_static_covariates( - pd.DataFrame({"basin_id": [1], "area": [100.0]}) - ) - - -class TestLSTMEncDecWithoutPastTarget: - def test_lstm_enc_dec_without_past_target( - self, synthetic_target, synthetic_past_cov, synthetic_future_cov - ): - """Verify model works when use_past_target=False. - - This tests that the encoder can operate purely on past covariates - without using past discharge values, which is useful for scenarios - where historical discharge is unreliable or missing. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_past_target=False, - n_epochs=2, - ) - - # Fit model - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Predict - predictions = model.predict(n=5, series=synthetic_target) - - # Verify output shape - assert predictions.n_timesteps == 5 - assert predictions.n_components == 1 - - -class TestLSTMEncDecWithAllCovariates: - def test_lstm_enc_dec_with_all_covariates( - self, synthetic_target_with_static, synthetic_past_cov, synthetic_future_cov - ): - """Verify full model works with all covariate types. - - Tests the complete model with: - - Past target (discharge) - - Past covariates (observed weather) - - Future covariates (forecasted weather) - - Static covariates (basin characteristics) - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=32, - n_layers=2, - dropout=0.1, - use_past_target=True, - use_learned_init=True, - use_skip_connection=True, - n_epochs=2, - ) - - # Fit model - model.fit( - series=synthetic_target_with_static, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Predict - predictions = model.predict(n=5, series=synthetic_target_with_static) - - # Verify output shape matches output_chunk_length - assert predictions.n_timesteps == 5 - assert predictions.n_components == 1 - - -class TestLearnedInitActivation: - def test_learned_init_activation( - self, synthetic_target_with_static, synthetic_past_cov, synthetic_future_cov - ): - """Verify learned hidden init architecture when use_learned_init=True. - - When static covariates are provided, the model should create learned - initialization networks for LSTM hidden/cell states that condition on - basin-specific features. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_learned_init=True, - n_epochs=2, - ) - - # Fit with static covariates attached to series - model.fit( - series=synthetic_target_with_static, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Static covariates should be extracted and used - assert model.model.static_cov_dim == 2 # basin_id and area - assert model.model.hidden_init is not None - assert model.model.cell_init is not None - - def test_learned_init_not_created_without_static_cov( - self, synthetic_target, synthetic_past_cov, synthetic_future_cov - ): - """Verify learned init is not created without static covariates. - - Even when use_learned_init=True, the initialization layers should - not be created if no static covariates are provided, since there's - no basin-specific information to condition on. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_learned_init=True, - n_epochs=2, - ) - - # Fit without static covariates - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Verify hidden_init is None - assert model.model.hidden_init is None - assert model.model.cell_init is None - - -class TestSkipConnectionActivation: - def test_skip_connection_activation( - self, synthetic_target, synthetic_past_cov, synthetic_future_cov - ): - """Verify skip connections are active when enabled. - - Tests that gated skip connection layers are created when - use_skip_connection=True. Skip connections help with gradient - flow and can improve learning of residual patterns. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_skip_connection=True, - n_epochs=2, - ) - - # Fit model - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Verify skip connection gates are created - assert model.model.encoder_gate is not None - assert model.model.decoder_gate is not None - - def test_skip_connection_not_created_when_disabled( - self, synthetic_target, synthetic_past_cov, synthetic_future_cov - ): - """Verify skip connections are not created when disabled. - - When use_skip_connection=False, the gate layers should be None, - saving parameters and computation. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_skip_connection=False, - n_epochs=2, - ) - - # Fit model - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Verify gates are None - assert model.model.encoder_gate is None - assert model.model.decoder_gate is None - - -class TestIgnoresPastTargetProperty: - def test_ignores_past_target_true_when_disabled(self): - """Verify property returns True when use_past_target=False. - - The ignores_past_target_values property communicates to Darts - whether the model needs historical discharge values. When False, - the framework knows to provide past target values during training. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - use_past_target=False, - ) - - assert model.ignores_past_target_values is True - - def test_ignores_past_target_false_when_enabled(self): - """Verify property returns False when use_past_target=True. - - When the model uses past target values (default), the property - should return False to ensure Darts provides historical discharge. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - use_past_target=True, - ) - - assert model.ignores_past_target_values is False - - -class TestModelRequiresFutureCovariates: - def test_model_requires_future_covariates( - self, synthetic_target, synthetic_past_cov - ): - """Verify model raises error without future covariates. - - The encoder-decoder architecture requires future covariates - (weather forecasts, snow model output) for the decoder. Training - should fail gracefully with a clear error if they're missing. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - n_epochs=1, - ) - - # Attempt to fit without future covariates should raise - with pytest.raises(ValueError, match="requires future covariates"): - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - verbose=False, - ) - - -class TestMultivariateTarget: - def test_multivariate_target_support( - self, synthetic_past_cov, synthetic_future_cov - ): - """Verify model can handle multivariate target series. - - Tests that the model can predict multiple discharge locations - simultaneously, which could be useful for multi-station forecasting. - """ - np.random.seed(45) - multivariate_target = TimeSeries.from_values( - np.random.randn(100, 3).astype(np.float32) - ) - - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=32, - n_layers=2, - dropout=0.1, - n_epochs=2, - ) - - # Fit and predict - model.fit( - series=multivariate_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - predictions = model.predict(n=5, series=multivariate_target) - - # Verify output has correct number of components - assert predictions.n_components == 3 - assert predictions.n_timesteps == 5 - - -class TestPredictionConsistency: - def test_predictions_are_deterministic_with_seed( - self, synthetic_target, synthetic_past_cov, synthetic_future_cov - ): - """Verify predictions are deterministic when random seed is set. - - For reproducibility in testing and debugging, predictions should - be identical when the same random seed is used. - """ - - def train_and_predict(seed): - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.0, # Disable dropout for deterministic predictions - n_epochs=2, - random_state=seed, - pl_trainer_kwargs={"enable_progress_bar": False}, - ) - - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - return model.predict(n=5, series=synthetic_target) - - # Train two models with same seed - pred1 = train_and_predict(seed=123) - pred2 = train_and_predict(seed=123) - - # Predictions should be identical - np.testing.assert_array_almost_equal(pred1.values(), pred2.values(), decimal=5) - - -class TestOutputChunkShift: - def test_output_chunk_shift_parameter( - self, synthetic_target, synthetic_past_cov, synthetic_future_cov - ): - """Verify output_chunk_shift parameter works correctly. - - The shift parameter allows forecasting further into the future, - e.g., predicting days 11-15 instead of days 1-5. This is useful - for extended-range forecasting scenarios. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - output_chunk_shift=10, # Forecast days 11-15 - hidden_size=16, - n_layers=1, - n_epochs=2, - ) - - # Should train without error - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Should predict without error - predictions = model.predict(n=5, series=synthetic_target) - assert predictions.n_timesteps == 5 - - -class TestStaticCovariatesProperty: - def test_supports_static_covariates_property(self): - """Verify model advertises static covariate support. - - The supports_static_covariates property tells Darts that the model - can utilize basin characteristics (area, elevation, etc.) in its - predictions. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - ) - - assert model.supports_static_covariates is True - - def test_supports_multivariate_property(self): - """Verify model advertises multivariate support. - - The supports_multivariate property indicates the model can handle - multiple target components simultaneously. - """ - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - ) - - assert model.supports_multivariate is True - - -class TestSingleLSTMMode: - """Tests for single-LSTM mode (no encoder) when there's no temporal past info.""" - - def test_encoder_not_created_without_past_info(self, synthetic_future_cov): - """Verify encoder is not created when use_past_target=False and no past covariates. - - When there's no temporal past information (no past target, no past covariates), - the encoder only has static features repeated across time, which provides no - useful temporal encoding. In this case, the encoder should be skipped entirely. - """ - np.random.seed(42) - target = TimeSeries.from_values(np.random.randn(100, 1).astype(np.float32)) - - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_past_target=False, - n_epochs=2, - ) - - # Fit without past covariates (only future covariates) - model.fit( - series=target, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Verify encoder components are None - assert model.model.needs_encoder is False - assert model.model.encoder_lstm is None - assert model.model.encoder_proj is None - assert model.model.encoder_gate is None - - # Decoder should still exist - assert model.model.decoder_lstm is not None - assert model.model.decoder_proj is not None - - def test_single_lstm_mode_trains_and_predicts(self, synthetic_future_cov): - """Verify single-LSTM mode model can train and generate predictions.""" - np.random.seed(42) - target = TimeSeries.from_values(np.random.randn(100, 1).astype(np.float32)) - - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_past_target=False, - n_epochs=2, - ) - - # Fit model (single-LSTM mode: no past covariates, no past target) - model.fit( - series=target, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Predict - predictions = model.predict(n=5, series=target) - - # Verify output shape - assert predictions.n_timesteps == 5 - assert predictions.n_components == 1 - - def test_single_lstm_mode_with_static_covariates(self, synthetic_future_cov): - """Verify single-LSTM mode uses learned init from static covariates. - - When static covariates are provided, the decoder should be initialized - with learned hidden states conditioned on basin characteristics. - """ - np.random.seed(42) - target = TimeSeries.from_values(np.random.randn(100, 1).astype(np.float32)) - target = target.with_static_covariates( - pd.DataFrame({"basin_id": [1], "area": [100.0]}) - ) - - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - dropout=0.1, - use_past_target=False, - use_learned_init=True, - n_epochs=2, - ) - - # Fit model (single-LSTM mode with static covariates) - model.fit( - series=target, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - # Verify encoder is None but learned init exists - assert model.model.needs_encoder is False - assert model.model.encoder_lstm is None - assert model.model.hidden_init is not None - assert model.model.cell_init is not None - - # Predict - predictions = model.predict(n=5, series=target) - assert predictions.n_timesteps == 5 - - def test_needs_encoder_flag_with_past_target( - self, synthetic_target, synthetic_future_cov - ): - """Verify needs_encoder=True when use_past_target=True.""" - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - use_past_target=True, - n_epochs=2, - ) - - model.fit( - series=synthetic_target, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - assert model.model.needs_encoder is True - assert model.model.encoder_lstm is not None - - def test_needs_encoder_flag_with_past_covariates( - self, synthetic_target, synthetic_past_cov, synthetic_future_cov - ): - """Verify needs_encoder=True when past_cov_dim > 0.""" - model = LSTMEncoderDecoderModel( - input_chunk_length=10, - output_chunk_length=5, - hidden_size=16, - n_layers=1, - use_past_target=False, - n_epochs=2, - ) - - model.fit( - series=synthetic_target, - past_covariates=synthetic_past_cov, - future_covariates=synthetic_future_cov, - verbose=False, - ) - - assert model.model.needs_encoder is True - assert model.model.encoder_lstm is not None diff --git a/tests/test_inference.py b/tests/test_inference.py index 5481c7e..443d72b 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -630,24 +630,6 @@ def test_loads_exolstm_model( mock_exolstm_class.load.assert_called_once_with(str(model_path)) assert result == mock_model - @patch("st_forecast.custom_models.ExoMamba.ExoMambaModel") - def test_loads_exomamba_model( - self, mock_exomamba_class, temp_run_folder, sample_config - ): - """Test that ExoMamba custom model is correctly loaded.""" - model_path = temp_run_folder / "test_model.pt" - model_path.touch() - - mock_model = Mock() - mock_exomamba_class.load.return_value = mock_model - - sample_config.model_type = "EXOMAMBA" - - result = load_model(sample_config, temp_run_folder) - - mock_exomamba_class.load.assert_called_once_with(str(model_path)) - assert result == mock_model - @patch("darts.models.TiDEModel") def test_model_type_case_insensitive( self, mock_tide_class, temp_run_folder, sample_config diff --git a/tests/test_model_initialization.py b/tests/test_model_initialization.py index 18d11c8..7e2c074 100644 --- a/tests/test_model_initialization.py +++ b/tests/test_model_initialization.py @@ -13,9 +13,9 @@ initialize_model, _get_model_class, _get_model_specific_kwargs, + _resolve_encoder, ) from st_forecast.custom_models.FANMixer import FANMixerModel -from st_forecast.custom_models.ExoMamba import ExoMambaModel from st_forecast.custom_models.ExoLSTM import ExoLSTMModel from st_forecast.pipeline.artifacts import RunConfig from st_forecast.training_components import LossLogger @@ -45,7 +45,6 @@ class TestGetModelClass: ("TSMixer", TSMixerModel), ("TiDE", TiDEModel), ("FANMixer", FANMixerModel), - ("ExoMamba", ExoMambaModel), ("ExoLSTM", ExoLSTMModel), ], ) @@ -95,7 +94,6 @@ def test_exolstm_kwargs(self, base_config): config.extra_hyperparams = {"n_lstm_layers": 3} kwargs = _get_model_specific_kwargs(config) assert kwargs["n_lstm_layers"] == 3 - assert kwargs["use_reversible_instance_norm"] is False class TestInitializeModel: @@ -177,3 +175,47 @@ def test_model_has_correct_chunk_lengths(self, base_config): model, _ = initialize_model(config) assert model.input_chunk_length == 20 assert model.output_chunk_length == 7 + + +class TestResolveEncoder: + def test_week(self): + assert _resolve_encoder("week") == {"cyclic": {"future": ["week"]}} + + def test_month(self): + assert _resolve_encoder("month") == {"cyclic": {"future": ["month"]}} + + def test_none(self): + assert _resolve_encoder("none") is None + + def test_invalid_raises(self): + with pytest.raises(ValueError, match="Unknown encoder"): + _resolve_encoder("day") + + +class TestEncoderConfig: + def _tsmixer_extras(self) -> dict: + return { + "num_blocks": 2, + "ff_size": 64, + "activation": "ReLU", + "normalize_before": True, + "norm_type": "LayerNorm", + } + + def test_default_encoder_is_month(self, base_config): + config = replace(base_config, model_type="TSMixer") + config.extra_hyperparams = self._tsmixer_extras() + model, _ = initialize_model(config) + assert model.add_encoders == {"cyclic": {"future": ["month"]}} + + def test_encoder_none_disables(self, base_config): + config = replace(base_config, model_type="TSMixer", encoder="none") + config.extra_hyperparams = self._tsmixer_extras() + model, _ = initialize_model(config) + assert model.add_encoders is None + + def test_encoder_month(self, base_config): + config = replace(base_config, model_type="TSMixer", encoder="month") + config.extra_hyperparams = self._tsmixer_extras() + model, _ = initialize_model(config) + assert model.add_encoders == {"cyclic": {"future": ["month"]}} diff --git a/tests/test_model_type_normalization.py b/tests/test_model_type_normalization.py index a4e6797..9d2f1a7 100644 --- a/tests/test_model_type_normalization.py +++ b/tests/test_model_type_normalization.py @@ -22,8 +22,6 @@ def test_canonical_forms_unchanged(self, canonical: str) -> None: ("tsmixer", "TSMixer"), ("FANMIXER", "FANMixer"), ("fanmixer", "FANMixer"), - ("EXOMAMBA", "ExoMamba"), - ("exomamba", "ExoMamba"), ("EXOLSTM", "ExoLSTM"), ("exolstm", "ExoLSTM"), ], diff --git a/tests/test_temporal_backends.py b/tests/test_temporal_backends.py deleted file mode 100644 index 2c63d1f..0000000 --- a/tests/test_temporal_backends.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Tests for TCN and Attention temporal backends.""" - -import pytest -import torch - -from st_forecast.custom_models.unified_temporal.backends.tcn import TCNBackend -from st_forecast.custom_models.unified_temporal.backends.attention import ( - AttentionBackend, -) - - -@pytest.fixture -def batch_tensor() -> torch.Tensor: - return torch.randn(4, 100, 64) # (B, L+T, hidden_size) - - -class TestTCNBackend: - def test_output_shape_matches_input(self, batch_tensor): - backend = TCNBackend(hidden_size=64) - output = backend(batch_tensor) - assert output.shape == batch_tensor.shape - - def test_default_kernel_sizes(self): - backend = TCNBackend(hidden_size=32) - assert backend.kernel_sizes == [15, 5, 3] - - def test_custom_kernel_sizes(self): - custom_kernels = [7, 3, 3, 3] - backend = TCNBackend(hidden_size=32, kernel_sizes=custom_kernels) - assert backend.kernel_sizes == custom_kernels - assert len(backend.blocks) == 4 - - @pytest.mark.parametrize("hidden_size", [32, 64, 128, 256]) - def test_various_hidden_sizes(self, hidden_size): - x = torch.randn(2, 50, hidden_size) - backend = TCNBackend(hidden_size=hidden_size) - output = backend(x) - assert output.shape == x.shape - - def test_dropout_applied(self): - backend = TCNBackend(hidden_size=64, dropout=0.5) - assert backend.dropout == 0.5 - - def test_dilations_increase_exponentially(self): - backend = TCNBackend(hidden_size=32, kernel_sizes=[3, 3, 3, 3]) - dilations = [block.conv1.dilation[0] for block in backend.blocks] - assert dilations == [1, 2, 4, 8] - - -class TestAttentionBackend: - def test_output_shape_matches_input(self, batch_tensor): - backend = AttentionBackend(hidden_size=64, num_heads=4) - output = backend(batch_tensor) - assert output.shape == batch_tensor.shape - - def test_default_parameters(self): - backend = AttentionBackend(hidden_size=64) - assert backend.num_heads == 4 - assert backend.num_layers == 2 - - def test_custom_num_layers(self): - backend = AttentionBackend(hidden_size=64, num_layers=4) - assert len(backend.layers) == 4 - - @pytest.mark.parametrize( - "hidden_size,num_heads", - [ - (32, 4), - (64, 8), - (128, 8), - (256, 16), - ], - ) - def test_various_configurations(self, hidden_size, num_heads): - x = torch.randn(2, 50, hidden_size) - backend = AttentionBackend(hidden_size=hidden_size, num_heads=num_heads) - output = backend(x) - assert output.shape == x.shape - - def test_dropout_applied(self): - backend = AttentionBackend(hidden_size=64, dropout=0.1) - assert backend.dropout == 0.1 - - def test_causal_attention_preserves_causality(self): - backend = AttentionBackend(hidden_size=32, num_heads=4, num_layers=1) - backend.eval() - - x1 = torch.randn(1, 10, 32) - x2 = x1.clone() - x2[0, 5:, :] = torch.randn(5, 32) # Change future tokens - - with torch.no_grad(): - out1 = backend(x1) - out2 = backend(x2) - - # First 5 positions should be identical (causal = can't see future) - torch.testing.assert_close(out1[0, :5, :], out2[0, :5, :], rtol=1e-4, atol=1e-4) - - -class TestBackendProtocolConformance: - """Test that both backends conform to TemporalBackend protocol.""" - - @pytest.mark.parametrize( - "backend_cls,kwargs", - [ - (TCNBackend, {"hidden_size": 64}), - (AttentionBackend, {"hidden_size": 64, "num_heads": 4}), - ], - ) - def test_forward_signature(self, backend_cls, kwargs, batch_tensor): - backend = backend_cls(**kwargs) - output = backend(batch_tensor) - assert isinstance(output, torch.Tensor) - assert output.shape == batch_tensor.shape - - @pytest.mark.parametrize( - "backend_cls,kwargs", - [ - (TCNBackend, {"hidden_size": 64}), - (AttentionBackend, {"hidden_size": 64, "num_heads": 4}), - ], - ) - def test_gradients_flow(self, backend_cls, kwargs): - x = torch.randn(2, 20, 64, requires_grad=True) - backend = backend_cls(**kwargs) - output = backend(x) - loss = output.sum() - loss.backward() - assert x.grad is not None - assert x.grad.shape == x.shape diff --git a/uv.lock b/uv.lock index 0003ad6..ef07e72 100644 --- a/uv.lock +++ b/uv.lock @@ -957,18 +957,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, ] -[[package]] -name = "mambapy" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "torch" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/92/4e/b0c2310e8bb6be1212aae87bb0493fc7b0d6b79efee86f9bdecebbf21739/mambapy-1.2.0.tar.gz", hash = "sha256:594944bf59dc148b739cc63b7f5aea44d41c94949be5b7293529a199883ff744", size = 37645, upload-time = "2024-07-31T12:00:28.855Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/fa/45873bcff2bd443a48210d4889da03d7355df97e760141e94b624a838b36/mambapy-1.2.0-py3-none-any.whl", hash = "sha256:37372ae2b13621beb5f7d60c486efdedcd21f79a2608e3d5f47b937f3196c282", size = 40117, upload-time = "2024-07-31T12:00:27.064Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -2752,7 +2740,6 @@ dependencies = [ { name = "darts" }, { name = "joblib" }, { name = "lightning" }, - { name = "mambapy" }, { name = "matplotlib" }, { name = "numpy" }, { name = "optuna" }, @@ -2784,7 +2771,6 @@ requires-dist = [ { name = "darts", specifier = "==0.43.0" }, { name = "joblib", specifier = ">=1.5.2" }, { name = "lightning", specifier = "==2.5.1" }, - { name = "mambapy", specifier = "==1.2.0" }, { name = "matplotlib", specifier = ">=3.10.6" }, { name = "numpy", specifier = ">=2.3.3" }, { name = "optuna", specifier = ">=4.5.0" }, From 752116f9d18097b92e5be8759c649de25fd4ad32 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Wed, 15 Apr 2026 15:29:47 +0200 Subject: [PATCH 05/20] config update --- configs/tft_kgz.json | 77 ----------------------------------------- configs/tide_kgz.json | 80 ------------------------------------------- 2 files changed, 157 deletions(-) delete mode 100644 configs/tft_kgz.json delete mode 100644 configs/tide_kgz.json diff --git a/configs/tft_kgz.json b/configs/tft_kgz.json deleted file mode 100644 index bc85366..0000000 --- a/configs/tft_kgz.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "region": "KGZ", - - "path_to_operational_forcing": "../../../data/kyg_data_forecast_tools/intermediate_data/control_member_forcing", - "path_to_hindcast_forcing": "../../../data/kyg_data_forecast_tools/intermediate_data/hindcast_forcing", - "HRU_forcing": "00003", - "path_rivers": "../../../data/kyg_data_forecast_tools/intermediate_data/runoff_day.csv", - "path_static": "../../../data/kyg_data_forecast_tools/config/models_and_scalers/static_features/ML_basin_attributes_v2.csv", - "path_to_sla": "../../../data/sla_silvan/fsc_sla_timeseries_gapfilled.csv", - "path_to_nir": "../../../data/sla_silvan/meanNIR_TS_allBasins.csv", - "path_to_swe": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/SWE", - "path_to_hs": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/HS", - "path_to_rof": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/RoF", - - "model_type": "TFT", - "model_name": "TFT_KGZ", - "encoder": "month", - "input_chunk_length": 30, - "forecast_horizon": 10, - - "exog_features": ["P", "T", "PET", "daylight_hours"], - "past_features": ["moving_avr_dis_3", "moving_avr_dis_5", "moving_avr_dis_10"], - "future_features": ["P", "T", "PET", "daylight_hours"], - - "moving_windows": [3, 5, 10], - "shift_moving_avg": false, - - "scale_discharge_bool": true, - "global_scaling": true, - "transform_log": false, - "transform_mm": true, - - "discharge_scaler_type": "standard", - "forcing_scaler_type": "standard", - "static_scaler_type": "standard", - - "n_epochs": 30, - "lr": 0.0005, - "weight_decay": 1e-5, - "hidden_size": 32, - "dropout": 0.3, - "batch_size": 1028, - "num_attention_heads": 4, - - "factor": 0.5, - "patience": 3, - "early_stopping_patience": 10, - - "scheduler_type": "reduce_on_plateau", - "scheduler_step_size": 10, - "scheduler_t_max": 30, - "scheduler_eta_min": 1e-5, - - "early_stopping_enabled": true, - "early_stopping_min_delta": 0.001, - - "likelihood_type": "quantile", - "gradient_clip_val": 0.1, - "optimizer": "Adam", - "accelerator": "auto", - "checkpoint_strategy": "last", - "mc_dropout": true, - - - "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95], - "num_samples": 200, - - "work_dir": "/Users/sandrohunziker/Documents/Sapphire_Logs/", - - "train_years": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016], - "val_years": [2017, 2018, 2019, 2020, 2021], - "test_years": [2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025], - - "codes_to_remove": [15049, 16093, 16681, 15960, 15954, 15213, 15217, 15261, 15292], - "unnatural_rivers": [15069, 15070, 15102, 15259, 17462, 16135, 16143, 16153, 16159], - "use_only_natural_rivers": false -} diff --git a/configs/tide_kgz.json b/configs/tide_kgz.json deleted file mode 100644 index 4cd8170..0000000 --- a/configs/tide_kgz.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "region": "KGZ", - - "path_to_operational_forcing": "../../../data/kyg_data_forecast_tools/intermediate_data/control_member_forcing", - "path_to_hindcast_forcing": "../../../data/kyg_data_forecast_tools/intermediate_data/hindcast_forcing", - "HRU_forcing": "00003", - "path_rivers": "../../../data/kyg_data_forecast_tools/intermediate_data/runoff_day.csv", - "path_static": "../../../data/kyg_data_forecast_tools/config/models_and_scalers/static_features/ML_basin_attributes_v2.csv", - "path_to_sla": "../../../data/sla_silvan/fsc_sla_timeseries_gapfilled.csv", - "path_to_nir": "../../../data/sla_silvan/meanNIR_TS_allBasins.csv", - "path_to_swe": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/SWE", - "path_to_hs": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/HS", - "path_to_rof": "../../../data/kyg_data_forecast_tools/intermediate_data/snow_data/RoF", - - "model_type": "TiDE", - "model_name": "TiDE_KGZ", - "encoder": "month", - "input_chunk_length": 30, - "forecast_horizon": 10, - - "exog_features": ["P", "T", "PET", "daylight_hours"], - "past_features": ["moving_avr_dis_3", "moving_avr_dis_5", "moving_avr_dis_10"], - "future_features": ["P", "T", "PET", "daylight_hours"], - - "moving_windows": [3, 5, 10], - "shift_moving_avg": false, - - "scale_discharge_bool": true, - "global_scaling": true, - "transform_log": false, - "transform_mm": true, - - "discharge_scaler_type": "standard", - "forcing_scaler_type": "standard", - "static_scaler_type": "standard", - - "n_epochs": 30, - "weight_decay": 4.3504e-05, - "lr": 0.0005, - "hidden_size": 64, - "dropout": 0.58, - "num_layers": 1, - "temporal_decoder_hidden_size": 32, - "temporal_hidden_size_past": 16, - "temporal_hidden_size_future": 16, - "decoder_output_dim": 16, - "batch_size": 1028, - - "factor": 0.5, - "patience": 3, - "early_stopping_patience": 10, - - "scheduler_type": "cosine", - "scheduler_step_size": 10, - "scheduler_t_max": 20, - "scheduler_eta_min": 5e-5, - - "early_stopping_enabled": true, - "early_stopping_min_delta": 0.001, - - "likelihood_type": "quantile", - "gradient_clip_val": 0.1, - "accelerator": "auto", - "checkpoint_strategy": "last", - "optimizer": "Adam", - - "quantiles": [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95], - "num_samples": 200, - "mc_dropout": true, - - "work_dir": "/Users/sandrohunziker/Documents/Sapphire_Logs/", - - "train_years": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016], - "val_years": [2017, 2018, 2019, 2020, 2021], - "test_years": [2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025], - - "codes_to_remove": [15049, 16093, 16681, 15960, 15954, 15213, 15217, 15261, 15292], - "unnatural_rivers": [15069, 15070, 15102, 15259, 17462, 16135, 16143, 16153, 16159], - "use_only_natural_rivers": false -} From 905562ef56957787d8a9a2720d5512b2c9ed9126 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:46:00 +0200 Subject: [PATCH 06/20] gauge id mess and evaluation --- st_forecast/data_utils/preparation.py | 9 +- st_forecast/evaluation/__init__.py | 2 + st_forecast/evaluation/evaluator.py | 28 +++++++ st_forecast/operational/sapphire_predictor.py | 12 +-- st_forecast/pipeline/workflow.py | 10 ++- tests/test_evaluation/test_evaluator.py | 83 +++++++++++++++++++ 6 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 tests/test_evaluation/test_evaluator.py diff --git a/st_forecast/data_utils/preparation.py b/st_forecast/data_utils/preparation.py index 0c8821d..07a331a 100644 --- a/st_forecast/data_utils/preparation.py +++ b/st_forecast/data_utils/preparation.py @@ -678,6 +678,9 @@ def _stack_covariates( ts.static_covariates[basin_id_col].values[0]: ts for ts in forcing_ts } + # Determine the ID column name used in the static_features index + static_id_col = static_features.index.name or "CODE" + result = [] for ts in discharge_ts: code = ts.static_covariates[basin_id_col].values[0] @@ -694,7 +697,11 @@ def _stack_covariates( stacked = stacked.stack(era5[feature]) if code in static_features.index: - stacked = stacked.with_static_covariates(static_features.loc[code]) + static_row = static_features.loc[code].copy() + # Add basin ID explicitly — .loc[code] drops the index value itself. + # extract_timeseries_components needs it to identify basins. + static_row[static_id_col] = code + stacked = stacked.with_static_covariates(static_row) result.append(stacked) diff --git a/st_forecast/evaluation/__init__.py b/st_forecast/evaluation/__init__.py index 32b09a7..cad0066 100644 --- a/st_forecast/evaluation/__init__.py +++ b/st_forecast/evaluation/__init__.py @@ -4,6 +4,7 @@ EvaluationConfig, EvaluationResult, evaluate, + ensure_forecast_step, filter_agricultural_period, align_predictions_observations, ) @@ -34,6 +35,7 @@ "EvaluationConfig", "EvaluationResult", "evaluate", + "ensure_forecast_step", "filter_agricultural_period", "align_predictions_observations", # Performance metrics diff --git a/st_forecast/evaluation/evaluator.py b/st_forecast/evaluation/evaluator.py index 3b6e5d2..9cceff4 100644 --- a/st_forecast/evaluation/evaluator.py +++ b/st_forecast/evaluation/evaluator.py @@ -41,6 +41,33 @@ class EvaluationResult: flood_global: pd.DataFrame +_ISSUE_DATE_CANDIDATES: tuple[str, ...] = ("forecast_date", "forecast_issue_date") + + +def ensure_forecast_step(df: pd.DataFrame) -> pd.DataFrame: + """Return `df` with a `forecast_step` column, deriving it if absent. + + When missing, `forecast_step` is computed as `(date - issue_date).days` + using whichever issue-date column is present: `forecast_date` + (SapphirePredictor) or `forecast_issue_date` (standard pipeline). + """ + if "forecast_step" in df.columns: + return df + + issue_col = next((c for c in _ISSUE_DATE_CANDIDATES if c in df.columns), None) + if issue_col is None: + raise ValueError( + "Cannot derive `forecast_step`: predictions DataFrame must have " + "either `forecast_step`, `forecast_date`, or `forecast_issue_date`." + ) + + df = df.copy() + df["date"] = pd.to_datetime(df["date"]) + df[issue_col] = pd.to_datetime(df[issue_col]) + df["forecast_step"] = (df["date"] - df[issue_col]).dt.days + return df + + def filter_agricultural_period( df: pd.DataFrame, date_col: str = "date" ) -> pd.DataFrame: @@ -131,6 +158,7 @@ def evaluate( agricultural_period: bool = False, ) -> EvaluationResult: """Main evaluation function.""" + predictions = ensure_forecast_step(predictions) merged_df = align_predictions_observations(predictions, observations) if agricultural_period: diff --git a/st_forecast/operational/sapphire_predictor.py b/st_forecast/operational/sapphire_predictor.py index f793e10..ac00b76 100644 --- a/st_forecast/operational/sapphire_predictor.py +++ b/st_forecast/operational/sapphire_predictor.py @@ -61,11 +61,10 @@ def __init__(self, model_path: str): self.static_df = pd.read_csv( static_path, index_col=self.config.static_id_col ) - # Ensure static_id_col exists as a column (needed for downstream use) + # Clean up legacy files that had CODE as both index and column static_id = self.config.static_id_col - if static_id not in self.static_df.columns: - self.static_df[static_id] = self.static_df.index - # Remove duplicate columns (e.g., CODE.1 from double-saving) + if static_id in self.static_df.columns: + self.static_df = self.static_df.drop(columns=[static_id]) dup_col = f"{static_id}.1" if dup_col in self.static_df.columns: self.static_df = self.static_df.drop(columns=[dup_col]) @@ -92,11 +91,6 @@ def __init__(self, model_path: str): static_features_list, self.config.static_scaler_type, ) - # Remove the CODE column so it doesn't pollute TimeSeries static covariates - if self.config.static_id_col in self.scaled_static_df.columns: - self.scaled_static_df = self.scaled_static_df.drop( - columns=[self.config.static_id_col] - ) else: self.scaled_static_df = None diff --git a/st_forecast/pipeline/workflow.py b/st_forecast/pipeline/workflow.py index f6dcdc4..247fb54 100644 --- a/st_forecast/pipeline/workflow.py +++ b/st_forecast/pipeline/workflow.py @@ -260,8 +260,14 @@ def _save_artifacts( ) scalers.save(output_path / "scalers") - # Save static features - static_df.to_csv(output_path / "static_df.csv") + # Save static features — drop the ID column if it duplicates the index + static_to_save = static_df.copy() + if ( + static_to_save.index.name + and static_to_save.index.name in static_to_save.columns + ): + static_to_save = static_to_save.drop(columns=[static_to_save.index.name]) + static_to_save.to_csv(output_path / "static_df.csv") # Save loss history if training_result.train_losses: diff --git a/tests/test_evaluation/test_evaluator.py b/tests/test_evaluation/test_evaluator.py new file mode 100644 index 0000000..801c037 --- /dev/null +++ b/tests/test_evaluation/test_evaluator.py @@ -0,0 +1,83 @@ +import pandas as pd +import pytest + +from st_forecast.evaluation.evaluator import ensure_forecast_step + + +class TestEnsureForecastStep: + def test_computes_from_forecast_date(self): + df = pd.DataFrame( + { + "date": pd.to_datetime( + ["2024-01-02", "2024-01-03", "2024-01-04"] + ), + "forecast_date": pd.to_datetime( + ["2024-01-01", "2024-01-01", "2024-01-01"] + ), + "code": [1, 1, 1], + "Q50": [10.0, 11.0, 12.0], + } + ) + + result = ensure_forecast_step(df) + + assert list(result["forecast_step"]) == [1, 2, 3] + + def test_computes_from_forecast_issue_date(self): + df = pd.DataFrame( + { + "date": pd.to_datetime( + ["2024-01-05", "2024-01-07"] + ), + "forecast_issue_date": pd.to_datetime( + ["2024-01-04", "2024-01-04"] + ), + "code": [1, 1], + "Q50": [10.0, 11.0], + } + ) + + result = ensure_forecast_step(df) + + assert list(result["forecast_step"]) == [1, 3] + + def test_preserves_existing_forecast_step(self): + df = pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-02", "2024-01-03"]), + "forecast_date": pd.to_datetime(["2024-01-01", "2024-01-01"]), + "forecast_step": [99, 99], + "code": [1, 1], + "Q50": [10.0, 11.0], + } + ) + + result = ensure_forecast_step(df) + + assert list(result["forecast_step"]) == [99, 99] + + def test_raises_when_no_issue_date_column(self): + df = pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-02", "2024-01-03"]), + "code": [1, 1], + "Q50": [10.0, 11.0], + } + ) + + with pytest.raises(ValueError, match="forecast_date"): + ensure_forecast_step(df) + + def test_accepts_string_date_columns(self): + df = pd.DataFrame( + { + "date": ["2024-01-02", "2024-01-04"], + "forecast_date": ["2024-01-01", "2024-01-01"], + "code": [1, 1], + "Q50": [10.0, 11.0], + } + ) + + result = ensure_forecast_step(df) + + assert list(result["forecast_step"]) == [1, 3] From 9251b3258d9161c3864b41b91b34554d81ad7878 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Thu, 30 Apr 2026 11:16:41 +0200 Subject: [PATCH 07/20] pre claude stuff --- st_forecast/{load_data_darts.py => DEP_load_data_darts.py} | 0 st_forecast/{train_model.py => DEP_train_model.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename st_forecast/{load_data_darts.py => DEP_load_data_darts.py} (100%) rename st_forecast/{train_model.py => DEP_train_model.py} (100%) diff --git a/st_forecast/load_data_darts.py b/st_forecast/DEP_load_data_darts.py similarity index 100% rename from st_forecast/load_data_darts.py rename to st_forecast/DEP_load_data_darts.py diff --git a/st_forecast/train_model.py b/st_forecast/DEP_train_model.py similarity index 100% rename from st_forecast/train_model.py rename to st_forecast/DEP_train_model.py From 68427d9f51478ea79a5e04d8971c6bd4358ed0d2 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Thu, 30 Apr 2026 13:56:08 +0200 Subject: [PATCH 08/20] slowly getting ready --- README.md | 2 +- examples/lamah_workflow.py | 6 +- pyproject.toml | 37 +- st_forecast/DEP_load_data_darts.py | 462 ------ st_forecast/DEP_train_model.py | 685 --------- st_forecast/__init__.py | 16 +- st_forecast/cli/train.py | 12 +- st_forecast/cli/tune.py | 11 +- st_forecast/data_utils/loaders.py | 32 +- st_forecast/data_utils/transformers.py | 8 +- st_forecast/evaluation/evaluate.py | 946 ------------- st_forecast/evaluation/sapphire_metrics.py | 1252 ----------------- st_forecast/get_data.py | 13 +- st_forecast/model_helper.py | 7 - st_forecast/operational/sapphire_predictor.py | 153 +- st_forecast/pipeline/forecast.py | 16 +- st_forecast/pipeline/inference.py | 62 +- st_forecast/pipeline/workflow.py | 6 + st_forecast/training_components.py | 25 + tests/test_inference.py | 47 +- tests/test_public_api_smoke.py | 431 ++++++ 21 files changed, 753 insertions(+), 3476 deletions(-) delete mode 100644 st_forecast/DEP_load_data_darts.py delete mode 100644 st_forecast/DEP_train_model.py delete mode 100644 st_forecast/evaluation/evaluate.py delete mode 100644 st_forecast/evaluation/sapphire_metrics.py create mode 100644 tests/test_public_api_smoke.py diff --git a/README.md b/README.md index a43b01b..4b26a25 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ result = train_model( External applications (e.g., the SAPPHIRE forecast system) use `SapphirePredictor` to run predictions from a trained model folder: ```python -from st_forecast.operational.sapphire_predictor import SapphirePredictor +from st_forecast import SapphirePredictor predictor = SapphirePredictor("runs/my_model") diff --git a/examples/lamah_workflow.py b/examples/lamah_workflow.py index c6afe05..2d1e3a5 100644 --- a/examples/lamah_workflow.py +++ b/examples/lamah_workflow.py @@ -142,8 +142,8 @@ def create_config(work_dir: Path) -> RunConfig: """Create RunConfig for LamaH example basins.""" return RunConfig( region="lamah_ce", - model_type="TiDE", - model_name="tide_lamah_example", + model_type="RNN", # Defaults to LSTM when using RNN + model_name="RNN_lamah_example", input_chunk_length=90, forecast_horizon=3, exog_features=[ @@ -156,8 +156,6 @@ def create_config(work_dir: Path) -> RunConfig: hidden_size=32, dropout=0.1, mc_dropout=True, - temporal_decoder_hidden_size=32, - decoder_output_dim=32, batch_size=258, lr=0.0005, work_dir=str(work_dir), diff --git a/pyproject.toml b/pyproject.toml index e9987b8..df62832 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,34 @@ [project] name = "st-forecast" version = "0.1.9" -description = "Add your description here" +description = "Deep-learning short-term discharge forecasting for multi-basin hydrological systems; entry point to the SAPPHIRE forecast tools." readme = "README.md" requires-python = ">=3.11" +authors = [ + { name = "Sandro Hunziker", email = "hunziker@hydrosolutions.ch" }, +] +keywords = [ + "hydrology", + "deep-learning", + "time-series", + "forecasting", + "discharge", + "darts", + "tide", + "tft", + "sapphire", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Hydrology", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + dependencies = [ "click>=8.0", "darts==0.43.0", @@ -23,6 +48,11 @@ dependencies = [ "tqdm>=4.67.1", ] +[project.urls] +Homepage = "https://github.com/sandrohuni/st-forecast" +Repository = "https://github.com/sandrohuni/st-forecast" +"SAPPHIRE Forecast Tools" = "https://github.com/hydrosolutions/SAPPHIRE_Forecast_Tools" + [project.scripts] st-forecast = "st_forecast.cli:main" @@ -43,3 +73,8 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["st_forecast"] + +[tool.pytest.ini_options] +markers = [ + "slow: end-to-end tests that train a real model (~5–30s)", +] diff --git a/st_forecast/DEP_load_data_darts.py b/st_forecast/DEP_load_data_darts.py deleted file mode 100644 index 0632465..0000000 --- a/st_forecast/DEP_load_data_darts.py +++ /dev/null @@ -1,462 +0,0 @@ -import pandas as pd -import numpy as np -from darts import TimeSeries -from st_forecast.utils import preprocessing_darts - - -def load_data( - use_only_natural_rivers=False, - train_start="2000-01-02", - train_end="2016-12-31", - test_start="2017-01-01", - test_end="2022-12-28", - input_chunk_length=60, - forecast_horizon=6, - save_scalers=False, - work_dir=r"../../../../code/machine_learning_in_hydrology/Models/GA", - exogene_features=[ - "P", - "T", - "PET", - "moving_avr_dis_5", - "moving_avr_dis_10", - "moving_avr_dis_3", - "daylight_hours", - "SWE", - ], - use_exp_moving_avg=False, - shift_moving_avg=False, - path_to_era5=r"../../../../data/forcing/ERA5_krg/HRU00003_forcing_2000-2023.csv", - path_to_static_features=r"../../../../GIS/ML_Sandro/ML_basin_attributes_v2.csv", - path_to_river_data=r"../../../../data/discharge/digitized_kyrgyz_hydromet/kyrgyz_hydromet_discharge_daily_2000_2023_kgz_filtered_v2.csv", -): - """ - params: - use_only_natural_rivers: bool, default False - If True, only natural rivers are used - train_start: str, - train_end: str, - test_start: str, - test_end: str, - input_chunk_length: int, default 60 - This controlls how much data is overlapping between the train and test data, so we can instantly predict in the test data. - forecast_horizon: int, default 6 - save_scalers: bool, default False - If True, scalers are saved - work_dir: str, - where the scalers are saved to - exogene_features: list, default ['P', 'T', 'PET' , 'moving_avr_dis_5', 'moving_avr_dis_10', 'moving_avr_dis_3','daylight_hours', 'SWE'] - Features to use as exogeneous features - use_exp_moving_avg: bool, default False - If True, the discharge data is transformed to exponential moving average - shift_moving_avg: bool, default False - If True, the moving averages are shifted by the forecast horizon - path_to_era5: str, default '../../../../data/forcing/ERA5_krg/HRU00003_forcing_2000-2023.csv' - Path to the ERA5 data - path_to_static_features: str, default '../../../../GIS/ML_Sandro/ML_basin_attributes_v2.csv' - Path to the static features - path_to_river_data: str, default '../../../../data/discharge/digitized_kyrgyz_hydromet/kyrgyz_hydromet_discharge_daily_2000_2023_kgz_filtered_v2.csv' - Path to the river data - """ - - # import era5 and static features, V2 with toktogul - df_era5 = pd.read_csv(path_to_era5, parse_dates=["date"]) - - basins = pd.read_csv(path_to_static_features) - - df_rivers = pd.read_csv(path_to_river_data) - - df_era5["code"] = df_era5["code"].astype(int) - df_rivers["code"] = df_rivers["code"].astype(int) - basins["CODE"] = basins["CODE"].astype(int) - - basins.index = basins["CODE"] - basins = basins.drop(columns=["cluster", "log_q"]) - - # filter out relevant basins - codes_to_remove = [16059, 15049] - if use_only_natural_rivers: - unnatural_rivers = [ - 15069, - 15070, - 15102, - 15259, - 17462, - 16135, - 16143, - 16153, - 16159, - ] - codes_to_drop = codes_to_remove + unnatural_rivers - else: - codes_to_drop = codes_to_remove - - codes_to_use = basins.index.values - codes_to_use = [code for code in basins.index.values if code not in codes_to_drop] - df_rivers = df_rivers[df_rivers["code"].isin(codes_to_use)] - df_era5 = df_era5[df_era5["code"].isin(codes_to_use)] - - # Toktogul Inflow river data has different time format - mask = df_rivers["code"] == 16936 - - dates_toktogul = df_rivers.loc[mask, "date"].values - - new_dates = [] - for d in dates_toktogul: - d_new = d[:10] - new_dates.append(d_new) - - df_rivers.loc[mask, "date"] = new_dates - - df_rivers["date"] = pd.to_datetime(df_rivers["date"]) - - # Sort the DataFrames by the 'date' column - df_rivers = df_rivers.sort_values(by="date") - df_era5 = df_era5.sort_values(by="date") - - # ----------------- # - # Train-Test Split - # ----------------- # - - # train the data into train and test - train_start = train_start - train_end = train_end - test_start = test_start - test_end = test_end - - input_chunk_length = input_chunk_length - forecast_horizon = forecast_horizon - - df_era5_train, df_era5_test = preprocessing_darts.split_data( - df_era5, - train_start, - train_end, - test_start, - test_end, - input_chunk_length, - forecast_horizon, - ) - df_rivers_train, df_rivers_test = preprocessing_darts.split_data( - df_rivers, - train_start, - train_end, - test_start, - test_end, - input_chunk_length, - forecast_horizon, - ) - - # ----------------- # - # Normalize the data - # ----------------- # - era5_to_scale = ["P", "T", "SWE", "PET", "daylight_hours", "melt"] - # scale all static features except the code column - static_to_scale = basins.columns.values[1:] - # normalize the data - scaler_type = "standard" - normalize_dict = preprocessing_darts.normalize_data( - discharge_train=df_rivers_train, - discharge_test=df_rivers_test, - era5_train=df_era5_train, - era5_test=df_era5_test, - basins=basins, - scaler_type=scaler_type, - values_to_scale=era5_to_scale, - static_features_to_scale=static_to_scale, - codes_to_use=codes_to_use, - ) - - df_rivers_train_scaled = normalize_dict["discharge_train"] - df_rivers_test_scaled = normalize_dict["discharge_test"] - era5_train_scaled = normalize_dict["era5_train"] - era5_test_scaled = normalize_dict["era5_test"] - static_features = normalize_dict["static_features"] - scalers = normalize_dict["scalers"] - era5_scalers = normalize_dict["era5_scalers"] - static_scalers = normalize_dict["static_scalers"] - - if save_scalers: - path_for_stats = work_dir - # save the scalers - preprocessing_darts.save_scalers( - path_for_stats, scalers, era5_scalers, static_scalers, scaler_type - ) - - # ----------------- # - # Add moving averages - # ----------------- # - moving_window = 10 - shifted_bool = shift_moving_avg - era5_train_scaled = preprocessing_darts.add_moving_avg( - discharge=df_rivers_train_scaled, - era5=era5_train_scaled, - shifted_bool=shifted_bool, - shift=forecast_horizon, - moving_window=moving_window, - ) - era5_test_scaled = preprocessing_darts.add_moving_avg( - discharge=df_rivers_test_scaled, - era5=era5_test_scaled, - shifted_bool=shifted_bool, - shift=forecast_horizon, - moving_window=moving_window, - ) - if shifted_bool: - era5_train_scaled = era5_train_scaled.rename( - columns={"shifted_moving_average_discharge": "moving_avr_dis_10_shifted"} - ) - era5_test_scaled = era5_test_scaled.rename( - columns={"shifted_moving_average_discharge": "moving_avr_dis_10_shifted"} - ) - else: - era5_train_scaled = era5_train_scaled.rename( - columns={"moving_average_discharge": "moving_avr_dis_10"} - ) - era5_test_scaled = era5_test_scaled.rename( - columns={"moving_average_discharge": "moving_avr_dis_10"} - ) - - # now for 5 days rolling average - era5_train_scaled = preprocessing_darts.add_moving_avg( - discharge=df_rivers_train_scaled, - era5=era5_train_scaled, - shifted_bool=shifted_bool, - shift=forecast_horizon, - moving_window=5, - ) - era5_test_scaled = preprocessing_darts.add_moving_avg( - discharge=df_rivers_test_scaled, - era5=era5_test_scaled, - shifted_bool=shifted_bool, - shift=forecast_horizon, - moving_window=5, - ) - if shifted_bool: - era5_train_scaled = era5_train_scaled.rename( - columns={"shifted_moving_average_discharge": "moving_avr_dis_5_shifted"} - ) - era5_test_scaled = era5_test_scaled.rename( - columns={"shifted_moving_average_discharge": "moving_avr_dis_5_shifted"} - ) - else: - era5_train_scaled = era5_train_scaled.rename( - columns={"moving_average_discharge": "moving_avr_dis_5"} - ) - era5_test_scaled = era5_test_scaled.rename( - columns={"moving_average_discharge": "moving_avr_dis_5"} - ) - - # now for 3 days rolling average - era5_train_scaled = preprocessing_darts.add_moving_avg( - discharge=df_rivers_train_scaled, - era5=era5_train_scaled, - shifted_bool=shifted_bool, - shift=forecast_horizon, - moving_window=3, - ) - era5_test_scaled = preprocessing_darts.add_moving_avg( - discharge=df_rivers_test_scaled, - era5=era5_test_scaled, - shifted_bool=shifted_bool, - shift=forecast_horizon, - moving_window=3, - ) - if shifted_bool: - era5_train_scaled = era5_train_scaled.rename( - columns={"shifted_moving_average_discharge": "moving_avr_dis_3_shifted"} - ) - era5_test_scaled = era5_test_scaled.rename( - columns={"shifted_moving_average_discharge": "moving_avr_dis_3_shifted"} - ) - else: - era5_train_scaled = era5_train_scaled.rename( - columns={"moving_average_discharge": "moving_avr_dis_3"} - ) - era5_test_scaled = era5_test_scaled.rename( - columns={"moving_average_discharge": "moving_avr_dis_3"} - ) - - # ----------------- # - # Transform the discharge to exponential moving average - # ----------------- # - span = 3 - if use_exp_moving_avg: - for code in df_rivers_train_scaled["code"].unique(): - mask = df_rivers_train_scaled["code"] == code - nan_mask = df_rivers_train_scaled.loc[:, "discharge"].isna().values - discharge = ( - df_rivers_train_scaled.loc[mask, "discharge"] - .ewm(span=span, adjust=False) - .mean() - ) - df_rivers_train_scaled.loc[mask, "discharge"] = discharge.values - df_rivers_train_scaled.loc[nan_mask, "discharge"] = np.nan - - for code in df_rivers_test_scaled["code"].unique(): - mask = df_rivers_test_scaled["code"] == code - nan_mask = df_rivers_test_scaled.loc[:, "discharge"].isna().values - discharge = ( - df_rivers_test_scaled.loc[mask, "discharge"] - .ewm(span=span, adjust=False) - .mean() - ) - df_rivers_test_scaled.loc[mask, "discharge"] = discharge.values - df_rivers_test_scaled.loc[nan_mask, "discharge"] = np.nan - - # ----------------- # - # Stack the ERA5 data on the discharge data - # ----------------- # - # Create time series - def stack_era5(discharge, forcing, exogene_features): - for i in range(len(discharge)): - code = discharge[i].static_covariates["code"].values[0] - - code_erats = forcing[i].static_covariates["code"].values[0] - era5 = forcing[i].slice_intersect(discharge[i]) - if code != code_erats: - print("error", code, code_erats) - - else: - for feature in exogene_features: - discharge[i] = discharge[i].stack(era5[feature]) - - # add the static covariates - discharge[i] = discharge[i].with_static_covariates( - static_features.loc[code] - ) - - return discharge - - discharge_train = TimeSeries.from_group_dataframe( - df_rivers_train_scaled, - group_cols="code", - time_col="date", - value_cols=["discharge"], - freq="1D", - ) - discharge_test = TimeSeries.from_group_dataframe( - df_rivers_test_scaled, - group_cols="code", - time_col="date", - value_cols=["discharge"], - freq="1D", - ) - - # select the exogene features - if shift_moving_avg: - # replace the moving average with the shifted one in the list - exogene_features = [ - x + "_shifted" - if x in ["moving_avr_dis_5", "moving_avr_dis_10", "moving_avr_dis_3"] - else x - for x in exogene_features - ] - - exogene_features_train = exogene_features - exogene_features_test = exogene_features_train - - forcing_train = TimeSeries.from_group_dataframe( - era5_train_scaled, - group_cols="code", - time_col="date", - value_cols=exogene_features_train, - freq="1D", - ) - forcing_test = TimeSeries.from_group_dataframe( - era5_test_scaled, - group_cols="code", - time_col="date", - value_cols=exogene_features_test, - freq="1D", - ) - discharge_train = stack_era5(discharge_train, forcing_train, exogene_features_train) - discharge_test = stack_era5(discharge_test, forcing_test, exogene_features_test) - - # ----------------- # - # Fill in small gaps - # ----------------- # - - from darts.utils.missing_values import extract_subseries, missing_values_ratio - - time_series_without_missing = [ - ts for ts in discharge_train if missing_values_ratio(ts["discharge"]) == 0 - ] - i = 0 - """ - Fill in gaps with len < 20 data points by inserting the dailymean of the year - """ - for ts in discharge_train: - gaps = ts.gaps("any") - - if len(gaps) > 0: - for row in gaps.iterrows(): - if row[1].values[2] < 20: - start_date = row[1].values[0] - end_date = row[1].values[1] - - dataframe = ts.pd_dataframe() - - # now calculate the mean of each dayoftheyear and fill the missing values with the mean - river_mean = dataframe.groupby(dataframe.index.dayofyear)[ - "discharge" - ].mean() - - dataframe_filtered = dataframe[ - (dataframe.index >= start_date) & (dataframe.index <= end_date) - ] - - for index, row in dataframe_filtered.iterrows(): - dataframe.at[index, "discharge"] = river_mean.loc[ - index.dayofyear - ] - dataframe["date"] = dataframe.index - ts = TimeSeries.from_dataframe( - dataframe, - time_col="date", - value_cols=["discharge"] + exogene_features_train, - static_covariates=ts.static_covariates, - freq="1D", - ) - - discharge_train[i] = ts - i += 1 - - # ----------------- # - # Slice the time series, so that there are no nan values in the input and output - # ----------------- # - discharge_train_sliced = [] - for i, ts in enumerate(discharge_train): - sliced_timeseries = extract_subseries(ts, min_gap_size=1, mode="any") - ratio = missing_values_ratio(ts) - for s in sliced_timeseries: - num_timesteps = s.n_timesteps - num_samples = s.n_samples - - if num_timesteps > input_chunk_length + forecast_horizon + 1: - discharge_train_sliced.append(s) - - discharge_test_sliced = [] - for i, ts in enumerate(discharge_test): - sliced_timeseries = extract_subseries(ts, min_gap_size=1, mode="any") - ratio = missing_values_ratio(ts) - for s in sliced_timeseries: - num_timesteps = s.n_timesteps - num_samples = s.n_samples - - # here we set it lower - if num_timesteps > input_chunk_length + forecast_horizon + 1: - discharge_test_sliced.append(s) - - # ----------------- # - # Return the data - # ----------------- # - - output_dict = { - "discharge_train_sliced": discharge_train_sliced, - "discharge_test_sliced": discharge_test_sliced, - "scalers": scalers, - "era5_scalers": era5_scalers, - "static_scalers": static_scalers, - } - - return output_dict diff --git a/st_forecast/DEP_train_model.py b/st_forecast/DEP_train_model.py deleted file mode 100644 index 2c3bd0a..0000000 --- a/st_forecast/DEP_train_model.py +++ /dev/null @@ -1,685 +0,0 @@ -import os -import json -import pandas as pd -import numpy as np - -import torch - - -from darts.models import TFTModel, TiDEModel, TSMixerModel -from st_forecast.pipeline.artifacts import normalize_model_type -from darts.utils.likelihood_models import QuantileRegression - -import logging -import random -# from tensorboard.backend.event_processing.event_accumulator import EventAccumulator - -from darts.utils.likelihood_models.base import LikelihoodType -from torch.optim import Adam -from torch.optim.lr_scheduler import ReduceLROnPlateau -from torch.nn.modules.loss import MSELoss -from torchmetrics.collections import MetricCollection - - -torch.serialization.add_safe_globals( - [ - QuantileRegression, - LikelihoodType, - Adam, - ReduceLROnPlateau, - MSELoss, - MetricCollection, - ] -) - - -# set logging level to warning and debug -logging.basicConfig(level=logging.WARNING) - -# Import the custom module -# try: -# from papercode import preprocessing_darts -# except ImportError as e: -# print(sys.path) -# raise ImportError(f"Failed to import module, check if it is in the same directory") from e - -# import load_data_darts -from st_forecast import get_data -from st_forecast import model_helper -from st_forecast import evaluate_model -from st_forecast import visualization -from st_forecast.data_utils.loaders import load_data_df, RawDataCollector -from st_forecast.data_utils.preparation import ( - prepare_train_data, - prepare_validation_data, -) -from st_forecast.custom_models.FANMixer import FANMixerModel -from st_forecast.custom_models.ExoLSTM import ExoLSTMModel - - -def _build_prepare_config(data_config: dict, model_config: dict) -> dict: - """Build config dict for prepare_train_data/prepare_validation_data. - - Merges relevant settings from both data and model configs into the - format expected by the preparation module. - """ - return { - "features": model_config.get("features", []), - "exog_features": model_config.get("exog_features", []), - "moving_windows": model_config.get("moving_windows", []), - "shift_moving_avg": model_config.get("shift_moving_avg", False), - "forecast_horizon": model_config.get("forecast_horizon", 6), - "input_chunk_length": model_config.get("input_chunk_length", 60), - "scale_discharge": model_config.get("scale_discharge_bool", True), - "discharge_scaler_type": model_config.get("discharge_scaler_type", "standard"), - "global_scaling": model_config.get("global_scaling", False), - "scale_forcing": model_config.get("scale_forcing", True), - "forcing_scaler_type": model_config.get("forcing_scaler_type", "minmax"), - "scale_static": model_config.get("scale_static", True), - "static_scaler_type": model_config.get("static_scaler_type", "minmax"), - "transform_mm": model_config.get("transform_mm", False), - "transform_log": model_config.get("transform_log", False), - "codes_to_remove": data_config.get("codes_to_remove", []), - "unnatural_rivers": data_config.get("unnatural_rivers", []), - "use_only_natural_rivers": model_config.get("use_only_natural_rivers", True), - } - - -def _load_data_new_pipeline(data_config: dict, model_config: dict) -> dict: - """Load data using the new prepare_train_data/prepare_validation_data functions. - - This implements the new modular data preparation pipeline that separates - training (fit scalers) from inference (apply scalers). - - Parameters - ---------- - data_config : dict - Data configuration dictionary with paths and split settings - model_config : dict - Model configuration with features and preprocessing settings - - Returns - ------- - dict - Same format as get_data.load_data_from_config() for compatibility: - { - 'discharge_train_sliced': list[TimeSeries], - 'discharge_val_sliced': list[TimeSeries] | None, - 'discharge_test_sliced': list[TimeSeries], - 'scalers': dict | None, - 'era5_scalers': dict, - 'static_scalers': dict, - 'split_mode': 'years' | 'dates' - } - """ - region = data_config.get("region", "UNKNOWN") - input_chunk_length = model_config.get("input_chunk_length", 60) - forecast_horizon = model_config.get("forecast_horizon", 6) - - # Step 1: Load raw data using RawDataCollector - collector = RawDataCollector(data_config, region) - raw_data = collector.collect_all() - - discharge_df = raw_data["discharge"] - forcing_df = raw_data["forcing"] - static_df = raw_data["static"] - - # Step 2: Combine discharge and forcing into a single temporal DataFrame - # The prepare functions expect a combined temporal_df - temporal_df = pd.merge(discharge_df, forcing_df, on=["date", "code"], how="inner") - - # Step 3: Split data temporally - use_year_split = "train_years" in data_config and "test_years" in data_config - - if use_year_split: - train_years = data_config["train_years"] - val_years = data_config.get("val_years") - test_years = data_config["test_years"] - - train_temporal, val_temporal, test_temporal = get_data.split_by_years( - temporal_df, - train_years, - val_years, - test_years, - input_chunk_length, - forecast_horizon, - ) - else: - train_start = data_config["train_start"] - train_end = data_config["train_end"] - test_start = data_config["test_start"] - test_end = data_config["test_end"] - - train_temporal, test_temporal = get_data.split_data( - temporal_df, - train_start, - train_end, - test_start, - test_end, - input_chunk_length, - forecast_horizon, - ) - val_temporal = None - - # Step 4: Build config for prepare functions - prepare_config = _build_prepare_config(data_config, model_config) - - # Step 5: Prepare training data (fits scalers) - train_result = prepare_train_data(train_temporal, static_df, prepare_config) - train_timeseries = train_result["timeseries"] - scalers = train_result["scalers"] - - # Step 6: Prepare validation data (applies scalers) - if val_temporal is not None: - val_result = prepare_validation_data( - val_temporal, static_df, scalers, prepare_config - ) - val_timeseries = val_result["timeseries"] - else: - val_timeseries = None - - # Step 7: Prepare test data (applies scalers) - test_result = prepare_validation_data( - test_temporal, static_df, scalers, prepare_config - ) - test_timeseries = test_result["timeseries"] - - # Return in the same format as the existing pipeline - return { - "discharge_train_sliced": train_timeseries, - "discharge_val_sliced": val_timeseries, - "discharge_test_sliced": test_timeseries, - "scalers": scalers.get("discharge"), - "era5_scalers": scalers.get("era5"), - "static_scalers": scalers.get("static"), - "split_mode": "years" if use_year_split else "dates", - } - - -# USAGE: uv run python -m st_forecast.train_model -def train_model(config_data_path, config_model_path): - """ - this function trains the model using the configuration file. - :param config_data_path: str: the path to the data configuration file - :param config_model_path: str: the path to the model configuration file - """ - # Set the random seed for reproducibility - random.seed(42) - torch.manual_seed(42) - np.random.seed(42) - - project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - with open(config_data_path, "r") as config_file: - config_data = json.load(config_file) - - with open(config_model_path, "r") as config_file: - config_model = json.load(config_file) - - # Set up the output directory - model_name = config_model[ - "model_name" - ] # how the model is called, and the name of the folder - region = config_data["region"] - folder_name = str(region) + "_" + str(model_name) - save_dir = config_data["save_dir"] - save_dir = os.path.join(save_dir, folder_name) - os.makedirs(save_dir, exist_ok=True) - - # Load config information - forecast_horizon = config_model["forecast_horizon"] - input_chunk_length = config_model["input_chunk_length"] - features = config_model["features"] - exog_features = config_model["exog_features"] - use_only_natural_rivers = config_model["use_only_natural_rivers"] - scale_discharge_bool = config_model["scale_discharge_bool"] - moving_windows = config_model["moving_windows"] - shift_moving_avg = config_model["shift_moving_avg"] - global_scaling = config_model.get("global_scaling", False) - transform_log = config_model.get("transform_log", False) - transform_mm = config_model.get("transform_mm", False) - - # Check whether to use the new modular prepare pipeline - use_new_prepare_pipeline = config_model.get("use_new_prepare_pipeline", False) - - if use_new_prepare_pipeline: - # Use the new prepare_train_data/prepare_validation_data functions - # This provides clean separation between training (fit scalers) and inference (apply scalers) - logging.info("Using new modular prepare pipeline") - data_dict = _load_data_new_pipeline( - data_config=config_data, model_config=config_model - ) - else: - # Use the existing load_data_from_config() function (default) - data_dict = get_data.load_data_from_config( - data_config=config_data, model_config=config_model - ) - - discharge_train_sliced = data_dict["discharge_train_sliced"] - discharge_val_sliced = data_dict.get("discharge_val_sliced") - discharge_test_sliced = data_dict["discharge_test_sliced"] - - past_features = config_model["past_features"] - future_features = config_model["future_features"] - - print("Preparing training, validation, and test sets \n") - print("Past features: ", past_features) - print("Future features: ", future_features) - - target_train = [q["discharge"] for q in discharge_train_sliced] - past_covariates_train = [q[past_features] for q in discharge_train_sliced] - future_covariates_train = [q[future_features] for q in discharge_train_sliced] - static_covariates_train = [q.static_covariates for q in discharge_train_sliced] - - target_test = [q["discharge"] for q in discharge_test_sliced] - past_covariates_test = [q[past_features] for q in discharge_test_sliced] - future_covariates_test = [q[future_features] for q in discharge_test_sliced] - static_covariates_test = [q.static_covariates for q in discharge_test_sliced] - - if discharge_val_sliced is not None: - print("Using separate validation set") - target_val = [q["discharge"] for q in discharge_val_sliced] - past_covariates_val = [q[past_features] for q in discharge_val_sliced] - future_covariates_val = [q[future_features] for q in discharge_val_sliced] - static_covariates_val = [q.static_covariates for q in discharge_val_sliced] - else: - print("Using training set for validation") - target_val = None - past_covariates_val = None - future_covariates_val = None - static_covariates_val = None - - def drop_code_static_covariates(ts): - static_new = ts.static_covariates.drop(columns=["CODE"]) - ts = ts.with_static_covariates(static_new) - return ts - - code_train = [ts.static_covariates["CODE"].values[0] for ts in target_train] - code_test = [ts.static_covariates["CODE"].values[0] for ts in target_test] - code_val = ( - [ts.static_covariates["CODE"].values[0] for ts in target_val] - if target_val is not None - else None - ) - - target_train = [drop_code_static_covariates(ts) for ts in target_train] - target_test = [drop_code_static_covariates(ts) for ts in target_test] - - # transform all series to float.32 - target_train = [ts.astype(np.float32) for ts in target_train] - past_covariates_train = [ts.astype(np.float32) for ts in past_covariates_train] - future_covariates_train = [ts.astype(np.float32) for ts in future_covariates_train] - - target_test = [ts.astype(np.float32) for ts in target_test] - past_covariates_test = [ts.astype(np.float32) for ts in past_covariates_test] - future_covariates_test = [ts.astype(np.float32) for ts in future_covariates_test] - - if target_val is not None: - target_val = [drop_code_static_covariates(ts) for ts in target_val] - target_val = [ts.astype(np.float32) for ts in target_val] - past_covariates_val = [ts.astype(np.float32) for ts in past_covariates_val] - future_covariates_val = [ts.astype(np.float32) for ts in future_covariates_val] - - ############ - # SETUP MODEL - ############ - model_type = normalize_model_type(config_model["model_type"]) - - if model_type == "TFT": - model, loss_logger = model_helper.initialize_TFT(config_model) - - elif model_type == "TSMixer": - model, loss_logger = model_helper.initialize_TSMixer(config_model) - - elif model_type == "TiDE": - model, loss_logger = model_helper.initialize_TiDE(config_model) - - elif model_type == "FANMixer": - model, loss_logger = model_helper.initialize_FANMixer(config_model) - - elif model_type == "ExoLSTM": - model, loss_logger = model_helper.initialize_ExoLSTM(config_model) - else: - raise ValueError(f"Model type {model_type} not recognized") - - ############ - # TRAIN MODEL - ############ - # choose validation source: prefer explicit validation split; fallback to test - val_series_used = target_val if target_val is not None else target_test - val_past_covariates_used = ( - past_covariates_val if past_covariates_val is not None else past_covariates_test - ) - val_future_covariates_used = ( - future_covariates_val - if future_covariates_val is not None - else future_covariates_test - ) - model.fit( - series=target_train, - past_covariates=past_covariates_train, - future_covariates=future_covariates_train, - val_series=val_series_used, - val_past_covariates=val_past_covariates_used, - val_future_covariates=val_future_covariates_used, - epochs=config_model["n_epochs"], - ) - - # Load the best model - work_dir = config_model["work_dir"] - if model_type == "TFT": - best_model = TFTModel.load_from_checkpoint( - model_name, work_dir=work_dir, best=True - ) - elif model_type == "TSMixer": - best_model = TSMixerModel.load_from_checkpoint( - model_name, work_dir=work_dir, best=True - ) - elif model_type == "TiDE": - best_model = TiDEModel.load_from_checkpoint( - model_name, work_dir=work_dir, best=True - ) - elif model_type == "FANMixer": - best_model = FANMixerModel.load_from_checkpoint( - model_name, work_dir=work_dir, best=True - ) - elif model_type == "ExoLSTM": - best_model = ExoLSTMModel.load_from_checkpoint( - model_name, work_dir=work_dir, best=True - ) - - use_swa = False - if use_swa: - best_model = model - - # save the model - best_model.save(os.path.join(save_dir, model_name + ".pt"), clean=True) - - # Define the log directory - log_dir = os.path.join(work_dir, model_name, "logs") - - # Extract training and validation loss - # train_steps, train_loss = extract_scalars(log_dir, 'train_loss') - # val_steps, val_loss = extract_scalars(log_dir, 'val_loss') - train_loss = loss_logger.get_train_losses() - val_loss = loss_logger.get_val_losses() - train_steps = np.arange(len(train_loss)) - val_steps = np.arange(len(val_loss)) - # Create pandas DataFrame - train_df = pd.DataFrame({"Step": train_steps, "Train Loss": train_loss}) - val_df = pd.DataFrame({"Step": val_steps, "Val Loss": val_loss}) - - # Save to CSV files - train_df.to_csv(os.path.join(save_dir, "train_loss.csv"), index=False) - val_df.to_csv(os.path.join(save_dir, "val_loss.csv"), index=False) - - # Plot the training and validation loss - visualization.make_train_vs_val_plot(train_df, val_df, output_dir=save_dir) - - ############ - # EVALUATE MODEL - ############ - - observed_path = config_data["path_rivers"] - observed = pd.read_csv(observed_path) - observed["date"] = pd.to_datetime(observed["date"], format="mixed") - observed["code"] = observed["code"].astype(int) - - # load static data - observed, df_era5, basins, success = load_data_df(path_config=config_data) - - if config_data["region"] == "TJK": - mask_river_date_to_nan = ( - (observed["code"] == 17325) - & (observed["date"] > "2020-12-31") - & (observed["date"] < "2022-01-01") - ) - observed.loc[mask_river_date_to_nan, "discharge"] = np.nan - observed["date"] = pd.to_datetime(observed["date"]) - - # drop duplicates on code and date - observed = observed.drop_duplicates(subset=["code", "date"], keep="last") - # Determine if we have a separate validation evaluation period - split_mode = data_dict.get("split_mode", "dates") - print( - "Evaluating model on validation and test sets \n" - if target_val is not None - else "Evaluating model on test set (no separate validation) \n" - ) - quantiles = [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95] - - # Validation evaluation (if available) - if target_val is not None: - predictions_val_df = evaluate_model.get_hindcast( - model=best_model, - target_series_list=target_val, - code_list=code_val, - past_covariates_list=past_covariates_val, - future_covariates_list=future_covariates_val, - scaler=data_dict["scalers"], - forecast_horizon=forecast_horizon, - probabilistic=True, - quantiles=quantiles, - mc_dropout=config_model.get("mc_dropout", False), - transform_mm=transform_mm, - transform_log=transform_log, - static_df=basins, - ) - predictions_val_df = predictions_val_df.round(2) - predictions_val_df.to_csv(os.path.join(save_dir, "predictions_val.csv")) - - # Derive validation date range - if ( - split_mode == "years" - and "val_years" in config_data - and config_data["val_years"] - ): - val_years = config_data["val_years"] - val_start = f"{min(val_years)}-01-01" - val_end = f"{max(val_years)}-12-31" - else: - # fallback: if only date ranges provided, reuse test range as no dedicated val - val_start = config_data.get("val_start", config_data.get("test_start")) - val_end = config_data.get("val_end", config_data.get("test_end")) - - metrics_val = evaluate_model.calculate_metrics( - predictions_df=predictions_val_df, - observed_df=observed, - start_date=val_start, - end_date=val_end, - probabilistic=True, - quantiles=quantiles, - agricultural_season=False, - ) - metrics_val.to_csv(os.path.join(save_dir, "metrics_val.csv")) - print("Metrics validation set \n") - print(metrics_val) - - predictions_df = evaluate_model.get_hindcast( - model=best_model, - target_series_list=target_test, - code_list=code_test, - past_covariates_list=past_covariates_test, - future_covariates_list=future_covariates_test, - scaler=data_dict["scalers"], - forecast_horizon=forecast_horizon, - probabilistic=True, - quantiles=quantiles, - mc_dropout=config_model.get("mc_dropout", False), - transform_mm=transform_mm, - transform_log=transform_log, - static_df=basins, - ) - - # round predictions to 2 decimals - predictions_df = predictions_df.round(2) - # save the predictions - predictions_df.to_csv(os.path.join(save_dir, "predictions.csv")) - - # Derive test date range - if split_mode == "years" and "test_years" in config_data: - test_years = config_data["test_years"] - test_start = f"{min(test_years)}-01-01" - test_end = f"{max(test_years)}-12-31" - else: - test_start = config_data["test_start"] - test_end = config_data["test_end"] - - metrics_test = evaluate_model.calculate_metrics( - predictions_df=predictions_df, - observed_df=observed, - start_date=test_start, - end_date=test_end, - probabilistic=True, - quantiles=quantiles, - agricultural_season=False, - ) - - print("Metrics test set \n") - print(metrics_test) - - # save the metrics - metrics_test.to_csv(os.path.join(save_dir, "metrics_test.csv")) - - if target_val is not None: - combined_predictions_df = pd.concat( - [predictions_val_df, predictions_df], axis=0 - ) - combined_predictions_df.to_csv( - os.path.join(save_dir, "predictions_val_test.csv") - ) - - ############ - # Evaluate on combined val + test - ############ - combined_start = min(val_start, test_start) - combined_end = max(val_end, test_end) - metrics_val_test = evaluate_model.calculate_metrics( - predictions_df=combined_predictions_df, - observed_df=observed, - start_date=combined_start, - end_date=combined_end, - probabilistic=True, - quantiles=quantiles, - agricultural_season=False, - ) - print("Metrics validation + test set \n") - print(metrics_val_test) - metrics_val_test.to_csv(os.path.join(save_dir, "metrics_val_test.csv")) - - else: - metrics_val_test = None - - if config_model["eval_on_train"] == True: - print("Evaluating model on train set \n") - predictions_train_df = evaluate_model.get_hindcast( - model=best_model, - target_series_list=target_train, - code_list=code_train, - past_covariates_list=past_covariates_train, - future_covariates_list=future_covariates_train, - scaler=data_dict["scalers"], - forecast_horizon=forecast_horizon, - probabilistic=True, - quantiles=quantiles, - mc_dropout=config_model.get("mc_dropout", False), - transform_mm=transform_mm, - transform_log=transform_log, - static_df=basins, - ) - - predictions_train_df = predictions_train_df.round(2) - - predictions_train_df.to_csv(os.path.join(save_dir, "predictions_train.csv")) - - # derive train date range - if split_mode == "years" and "train_years" in config_data: - tr_years = config_data["train_years"] - train_start = f"{min(tr_years)}-01-01" - train_end = f"{max(tr_years)}-12-31" - else: - train_start = config_data["train_start"] - train_end = config_data["train_end"] - - metrics_train = evaluate_model.calculate_metrics( - predictions_df=predictions_train_df, - observed_df=observed, - start_date=train_start, - end_date=train_end, - probabilistic=True, - quantiles=quantiles, - agricultural_season=False, - ) - metrics_train.to_csv(os.path.join(save_dir, "metrics_train.csv")) - - ############ - # Generate plots NSE, nRMSE, MAE, and QuantileLoss - ############ - metrics_plot = ["NSE", "nRMSE", "MAE", "MAPE", "QuantileLoss"] - for metric in metrics_plot: - file_name_train = metric + "_train.png" - if config_model["eval_on_train"] == True: - visualization.plot_metric_boxplot_per_step( - df=metrics_train, - metric=metric, - save_dir=save_dir, - filename=file_name_train, - hue=None, - plot_mean=True, - ) - if target_val is not None: - file_name_val = metric + "_val.png" - visualization.plot_metric_boxplot_per_step( - df=metrics_val, - metric=metric, - save_dir=save_dir, - filename=file_name_val, - hue=None, - plot_mean=True, - ) - file_name_test = metric + "_test.png" - visualization.plot_metric_boxplot_per_step( - df=metrics_test, - metric=metric, - save_dir=save_dir, - filename=file_name_test, - hue=None, - plot_mean=True, - ) - - if metrics_val_test is not None: - file_name_val_test = metric + "_val_test.png" - visualization.plot_metric_boxplot_per_step( - df=metrics_val_test, - metric=metric, - save_dir=save_dir, - filename=file_name_val_test, - hue=None, - plot_mean=True, - ) - - ############ - # Save the results and generate output file - ############ - # Save the configuration files - with open(os.path.join(save_dir, "config_data.json"), "w") as config_file: - json.dump(config_data, config_file, indent=4) - - with open(os.path.join(save_dir, "config_model.json"), "w") as config_file: - json.dump(config_model, config_file, indent=4) - - name_of_file = r"/" + model_name + r"_description.txt" - - with open(save_dir + name_of_file, "w") as f: - f.write(model_name + "\n") - f.write("saved in folder: {}".format(save_dir) + "\n") - if config_model["eval_on_train"] == True: - f.write("Train Results \n: {} \n".format(metrics_train.describe())) - f.write("Test Results \n: {} \n".format(metrics_test.describe())) - - -if __name__ == "__main__": - config_data_path = "st_forecast/config/data_KGZ.json" - # config_model_path = "pentad_decad_forecast/config/TSMIXER_decad.json" - config_model_path = "st_forecast/config/ExoLSTM_decad.json" - train_model(config_data_path, config_model_path) diff --git a/st_forecast/__init__.py b/st_forecast/__init__.py index ea51657..f0a8fa9 100644 --- a/st_forecast/__init__.py +++ b/st_forecast/__init__.py @@ -4,7 +4,13 @@ """ from st_forecast.config.data_config import ColumnConfig +from st_forecast.operational.sapphire_predictor import ( + SapphirePredictionError, + SapphirePredictor, + silence_pl_warnings, +) from st_forecast.pipeline.artifacts import RunConfig +from st_forecast.pipeline.inference import DEFAULT_QUANTILES from st_forecast.pipeline.workflow import TrainModelResult, train_model __version__ = "0.1.9" @@ -15,11 +21,15 @@ TARGET_COL = "discharge" __all__ = [ + "BASIN_ID_COL", "ColumnConfig", + "DATE_COL", + "DEFAULT_QUANTILES", "RunConfig", + "SapphirePredictionError", + "SapphirePredictor", + "TARGET_COL", "TrainModelResult", + "silence_pl_warnings", "train_model", - "BASIN_ID_COL", - "DATE_COL", - "TARGET_COL", ] diff --git a/st_forecast/cli/train.py b/st_forecast/cli/train.py index 15bf600..4af6ff4 100644 --- a/st_forecast/cli/train.py +++ b/st_forecast/cli/train.py @@ -1,13 +1,11 @@ """CLI command for training the forecasting model.""" import logging -import random import shutil import click import numpy as np import pandas as pd -import torch from pathlib import Path from st_forecast import get_data @@ -19,17 +17,11 @@ ) from st_forecast.pipeline.artifacts import Scalers, load_run_artifacts from st_forecast.pipeline.training import train +from st_forecast.training_components import set_seed logger = logging.getLogger(__name__) -def _set_seeds(seed: int) -> None: - """Set random seeds for reproducibility.""" - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - - def _save_losses( train_losses: list[float], val_losses: list[float], @@ -77,7 +69,7 @@ def train_cli(run_folder: Path, config: Path | None, seed: int) -> None: shutil.copy(config, config_dest) logger.info(f"Copied config from {config} to {config_dest}") - _set_seeds(seed) + set_seed(seed) logger.info(f"Random seed set to {seed}") # Load artifacts diff --git a/st_forecast/cli/tune.py b/st_forecast/cli/tune.py index d62bfe9..3ff62c9 100644 --- a/st_forecast/cli/tune.py +++ b/st_forecast/cli/tune.py @@ -2,7 +2,6 @@ import json import logging -import random import shutil from pathlib import Path @@ -10,7 +9,6 @@ import numpy as np import optuna import pandas as pd -import torch from st_forecast import get_data from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector @@ -21,18 +19,13 @@ ) from st_forecast.pipeline.artifacts import RunConfig, Scalers from st_forecast.pipeline.training import TrainingResult, train +from st_forecast.training_components import set_seed from st_forecast.tuning.config import TuneConfig from st_forecast.tuning.objective import create_objective logger = logging.getLogger(__name__) -def _set_seeds(seed: int) -> None: - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - - def _load_and_prepare_data( config: RunConfig, ) -> tuple[dict, pd.DataFrame, dict]: @@ -244,7 +237,7 @@ def tune_cli(run_folder: Path, config: Path, seed: int, resume: bool) -> None: shutil.copy(config, config_dest) logger.info(f"Copied config from {config} to {config_dest}") - _set_seeds(seed) + set_seed(seed) logger.info(f"Random seed set to {seed}") with open(config) as f: diff --git a/st_forecast/data_utils/loaders.py b/st_forecast/data_utils/loaders.py index 2a56313..58b3223 100644 --- a/st_forecast/data_utils/loaders.py +++ b/st_forecast/data_utils/loaders.py @@ -240,8 +240,8 @@ def _merge_optional(path_key: str, var_name: str): df_tmp[date_col] = pd.to_datetime(df_tmp[date_col]) nonlocal df_era5 df_era5 = pd.merge(df_era5, df_tmp, on=[date_col, basin_id_col], how="left") - except Exception as e: - print(f"Error loading {var_name} data from {p}: {e}") + except Exception: + logger.exception(f"Error loading {var_name} data from {p}") success_sla = False # merge optional snow / runoff factors @@ -260,8 +260,8 @@ def _merge_optional(path_key: str, var_name: str): df_era5, sla_df, on=[date_col, basin_id_col], how="left" ) success_sla = True - except Exception as e: - print(f"Error loading SLA/NIR data: {e}") + except Exception: + logger.exception("Error loading SLA/NIR data") success_sla = False # enforce dtypes @@ -366,8 +366,8 @@ def load_data_df( success["sca"] = True - except Exception as e: - print(f"Error loading SCA data: {str(e)}") + except Exception: + logger.exception("Error loading SCA data") success["sca"] = False # SWE DATA @@ -394,8 +394,8 @@ def load_data_df( forcing = pd.merge(forcing, swe_df, on=[date_col, basin_id_col], how="left") success["swe"] = True - except Exception as e: - print(f"Error loading SWE data: {str(e)}") + except Exception: + logger.exception("Error loading SWE data") success["swe"] = False # HS DATA @@ -416,8 +416,8 @@ def load_data_df( forcing = pd.merge(forcing, hs_df, on=[date_col, basin_id_col], how="left") success["hs"] = True - except Exception as e: - print(f"Error loading HS data: {str(e)}") + except Exception: + logger.exception("Error loading HS data") success["hs"] = False # ROF DATA @@ -437,8 +437,8 @@ def load_data_df( rof_df = rof_df.drop_duplicates(subset=[date_col, basin_id_col]) forcing = pd.merge(forcing, rof_df, on=[date_col, basin_id_col], how="left") success["rof"] = True - except Exception as e: - print(f"Error loading ROF data: {str(e)}") + except Exception: + logger.exception("Error loading ROF data") success["rof"] = False path_to_sla = path_config.get("path_to_sla") @@ -466,8 +466,8 @@ def load_data_df( try: sla_df = append_nir_to_sla(sla_df, path_to_nir) columns_of_interest.append("NIR") - except Exception as e: - print(f"Error appending NIR data to SLA data: {str(e)}") + except Exception: + logger.exception("Error appending NIR data to SLA data") sla_df = sla_df.rename( columns={"gla_fsc": "gla_fsc_total", "fsc": "fsc_basin"} @@ -532,8 +532,8 @@ def safe_convert_to_int(value): success["sla"] = True - except Exception as e: - print(f"Error loading SLA data: {str(e)}") + except Exception: + logger.exception("Error loading SLA data") success["sla"] = False return discharge, forcing, static, success diff --git a/st_forecast/data_utils/transformers.py b/st_forecast/data_utils/transformers.py index d3fdb59..299489e 100644 --- a/st_forecast/data_utils/transformers.py +++ b/st_forecast/data_utils/transformers.py @@ -190,13 +190,13 @@ def time_shift_sla_data( # Convert 'date' column to datetime if not already df[date_col] = pd.to_datetime(df[date_col]) unique_days_of_month = df[date_col].dt.day.unique() - print(f"Unique days of month in SLA data: {unique_days_of_month}") + logger.debug(f"Unique days of month in SLA data: {unique_days_of_month}") # Shift the date based on the day of the month df.loc[df[date_col].dt.day.isin([1, 11]), date_col] += pd.DateOffset(days=9) df.loc[df[date_col].dt.day == 21, date_col] = df[date_col] + pd.offsets.MonthEnd() unique_days_of_month = df[date_col].dt.day.unique() - print(f"Unique days of month after SLA data shift: {unique_days_of_month}") + logger.debug(f"Unique days of month after SLA data shift: {unique_days_of_month}") return df @@ -308,8 +308,8 @@ def load_sla_nir_data( sla_df, path_to_nir, date_col=date_col, basin_id_col=basin_id_col ) columns_of_interest.append("NIR") - except Exception as e: - print(f"Error appending NIR data to SLA data: {str(e)}") + except Exception: + logger.exception("Error appending NIR data to SLA data") sla_df = sla_df.rename(columns={"gla_fsc": "gla_fsc_total", "fsc": "fsc_basin"}) diff --git a/st_forecast/evaluation/evaluate.py b/st_forecast/evaluation/evaluate.py deleted file mode 100644 index 4d9ce86..0000000 --- a/st_forecast/evaluation/evaluate.py +++ /dev/null @@ -1,946 +0,0 @@ -import pandas as pd -import numpy as np -import os -import datetime - -from evaluation import metric_functions - - -def days_until_next_forecast_pentad(date, hindcast_dates): - """ - This function returns a boolean value, and the number of days until the next forecast. - """ - days_to_save = [5, 10, 15, 20, 25] - # Convert NumPy datetime64 to Python datetime - date = pd.to_datetime(date) - - # check if today is the last day of the month - tomorrow = date + datetime.timedelta(days=1) - is_last_day_of_month = tomorrow.month != date.month - - # check if today is in the list of days to save - is_day_to_save = date.day in days_to_save - - make_forecast = is_last_day_of_month or is_day_to_save - - # days until next forecast - if is_last_day_of_month: - days_until_next_forecast = 5 - - elif date.day in [5, 10, 15, 20]: - days_until_next_forecast = 5 - - else: - # if the month has 30 days -> 5 days - # if the month has 31 days -> 6 days - # if the month has 28 days -> 3 days - # if the month has 29 days -> 4 days - - if date.month in [1, 3, 5, 7, 8, 10, 12]: - days_until_next_forecast = 6 - elif date.month in [2]: - # check if the year is a leap year - if date.year % 4 == 0: - days_until_next_forecast = 4 - else: - days_until_next_forecast = 3 - else: - days_until_next_forecast = 5 - - if not make_forecast: - days_until_next_forecast = 0 - - return make_forecast, days_until_next_forecast - - -def days_until_next_forecast_decad(date, hindcast_dates): - """ - This function returns a boolean value, and the number of days until the next forecast. - """ - days_to_save = [10, 20] - # Convert NumPy datetime64 to Python datetime - date = pd.to_datetime(date) - - # check if today is the last day of the month - tomorrow = date + datetime.timedelta(days=1) - is_last_day_of_month = tomorrow.month != date.month - - # check if today is in the list of days to save - is_day_to_save = date.day in days_to_save - - make_forecast = is_last_day_of_month or is_day_to_save - - # days until next forecast - if is_last_day_of_month: - days_until_next_forecast = 10 - - elif date.day in [10]: - days_until_next_forecast = 10 - - else: - # if the month has 30 days -> 10 days - # if the month has 31 days -> 11 days - # if the month has 28 days -> 8 days - # if the month has 29 days -> 9 days - if date.month in [1, 3, 5, 7, 8, 10, 12]: - days_until_next_forecast = 11 - elif date.month in [2]: - # check if the year is a leap year - if date.year % 4 == 0: - days_until_next_forecast = 8 - else: - days_until_next_forecast = 9 - else: - days_until_next_forecast = 10 - - if not make_forecast: - days_until_next_forecast = 0 - - return make_forecast, days_until_next_forecast - - -def process_date(df): - df = df.copy() - df["date"] = pd.to_datetime(df["date"], format="mixed") - - # code to integer - df["code"] = df["code"].astype(int) - - columns = df.columns - if "forecast_date" in columns: - df["forecast_date"] = pd.to_datetime(df["forecast_date"], format="mixed") - df = df.drop_duplicates(subset=["forecast_date", "date", "code"], keep="last") - df = df.sort_values(by=["forecast_date", "date"]) - else: - df = df.drop_duplicates(subset=["date", "code"], keep="last") - df = df.sort_values(by="date") - return df - - -def aggregate_data(df): - df = df.copy() - df_aggregated = pd.DataFrame() - for code in df["code"].unique(): - # iterate through the unique forecast dates - for date in df["forecast_date"].unique(): - # select the data for the code and forecast date - df_code_date = df[ - (df["code"] == code) & (df["forecast_date"] == date) - ].copy() - # calculate the mean discharge - if len(df_code_date) == 0: - continue - - days_ = df_code_date["days_until_next_forecast"].values[0] - actual_length = len(df_code_date) - # Use the minimum between available days and days_until_next_forecast - days_to_use = min(days_, actual_length) - - discharge_cols = [col for col in df_code_date.columns if "Q" in col] - new_row = {} - for col in discharge_cols: - new_row[col] = df_code_date[col].iloc[:days_].mean() - - new_row["code"] = code - new_row["forecast_date"] = date - new_row["days_until_next_forecast"] = days_ - new_row["date"] = df_code_date["date"].iloc[0] - - new_row = pd.DataFrame(new_row, index=[0]) - - # add the new row to the aggregated dataframe - df_aggregated = pd.concat( - [df_aggregated, new_row], axis=0, ignore_index=True - ) - - return df_aggregated - - -def process_model_data( - predictions_path, - intermediate_data_path, - model_name, - return_daily_data=False, - eval_on_assimilated=False, - add_path=None, -): - # Load the data - hindcast_path = os.path.join(predictions_path, model_name) - if model_name in ["Mamba", "Mamba_Assim", "LSTM", "LSTM_Assim"]: - pentad = pd.read_csv(add_path, parse_dates=["date"]) - decad = pd.read_csv(add_path, parse_dates=["date"]) - else: - pentad = pd.read_csv( - os.path.join(hindcast_path, f"pentad_{model_name}_forecast.csv"), - parse_dates=["date"], - ) - decad = pd.read_csv( - os.path.join(hindcast_path, f"decad_{model_name}_forecast.csv"), - parse_dates=["date"], - ) - - if model_name in ["Mamba", "Mamba_Assim", "LSTM", "LSTM_Assim", "ARIMA"]: - # rename Q to Q50 - if eval_on_assimilated: - pentad = pentad.rename(columns={"assimilated_forecast": "Q50"}) - decad = decad.rename(columns={"assimilated_forecast": "Q50"}) - else: - pentad = pentad.rename(columns={"Q": "Q50"}) - decad = decad.rename(columns={"Q": "Q50"}) - - hydromet_discharge = pd.read_csv( - os.path.join(intermediate_data_path, "runoff_day.csv") - ) - linear_regression_pentad = pd.read_csv( - os.path.join(intermediate_data_path, "forecast_pentad_linreg.csv") - ) - linear_regression_decad = pd.read_csv( - os.path.join(intermediate_data_path, "forecast_decad_linreg.csv") - ) - - # Process the date - pentad = process_date(pentad) - decad = process_date(decad) - hydromet_discharge = process_date(hydromet_discharge) - linear_regression_pentad = process_date(linear_regression_pentad) - linear_regression_decad = process_date(linear_regression_decad) - - # get the same codes - codes = pentad["code"].unique() - hydromet_discharge = hydromet_discharge[hydromet_discharge["code"].isin(codes)] - linear_regression_pentad = linear_regression_pentad[ - linear_regression_pentad["code"].isin(codes) - ] - - print( - f"Codes in linear_regression_pentad: {linear_regression_pentad['code'].unique()}" - ) - - codes_decad = decad["code"].unique() - hydromet_discharge = hydromet_discharge[ - hydromet_discharge["code"].isin(codes_decad) - ] - linear_regression_decad = linear_regression_decad[ - linear_regression_decad["code"].isin(codes_decad) - ] - - print( - f"Codes in linear_regression_decad: {linear_regression_decad['code'].unique()}" - ) - - # selct time period 2017 - 2020 for linear regression - # select the same dates - dates_pentad = linear_regression_pentad["date"].unique() - set_dates = set(dates_pentad) - set_dates = set_dates.intersection(set(pentad["forecast_date"].unique())) - linear_regression_pentad = linear_regression_pentad[ - linear_regression_pentad["date"].isin(set_dates) - ] - pentad = pentad[pentad["forecast_date"].isin(set_dates)] - - dates_decad = linear_regression_decad["date"].unique() - set_dates = set(dates_decad) - set_dates = set_dates.intersection(set(decad["forecast_date"].unique())) - linear_regression_decad = linear_regression_decad[ - linear_regression_decad["date"].isin(set_dates) - ] - decad = decad[decad["forecast_date"].isin(set_dates)] - - # return the daily data - if return_daily_data: - # drop duplicates on forecast date and code and date - pentad = pentad.drop_duplicates( - subset=["forecast_date", "code", "date"], keep="last" - ) - decad = decad.drop_duplicates( - subset=["forecast_date", "code", "date"], keep="last" - ) - return ( - pentad, - decad, - hydromet_discharge, - linear_regression_pentad, - linear_regression_decad, - ) - - forecast_dates = pentad["forecast_date"].values - days_until_next_forecast_list = [] - for date in forecast_dates: - make_forecast, days_ = days_until_next_forecast_pentad(date, forecast_dates) - days_until_next_forecast_list.append(days_) - - print( - "Average days until next forecast pentad: ", - np.mean(days_until_next_forecast_list), - ) - - pentad["days_until_next_forecast"] = days_until_next_forecast_list - - forecast_dates = decad["forecast_date"].values - days_until_next_forecast_list = [] - for date in forecast_dates: - make_forecast, days_ = days_until_next_forecast_decad(date, forecast_dates) - days_until_next_forecast_list.append(days_) - - print( - "Average days until next forecast decad: ", - np.mean(days_until_next_forecast_list), - ) - decad["days_until_next_forecast"] = days_until_next_forecast_list - - # Aggregate the data - pentad = aggregate_data(pentad) - decad = aggregate_data(decad) - - # drop duplicates on forecast date and code - pentad = pentad.drop_duplicates(subset=["forecast_date", "code"], keep="last") - decad = decad.drop_duplicates(subset=["forecast_date", "code"], keep="last") - - return ( - pentad, - decad, - hydromet_discharge, - linear_regression_pentad, - linear_regression_decad, - ) - - -def create_ensemble(prediction_dfs, columns_to_aggregate): - """ - Create an ensemble dataframe from multiple prediction dataframes using a merge-based approach. - - Parameters: - ----------- - prediction_dfs : list - List of pandas DataFrames containing predictions - columns_to_aggregate : list - List of column names to aggregate (e.g., ['Q1', 'Q2', 'Q3']) - - Returns: - -------- - pandas.DataFrame - Ensemble dataframe with averaged predictions - """ - if not prediction_dfs or len(prediction_dfs) == 0: - raise ValueError("No prediction dataframes provided") - - # Deep copy the dataframes to avoid modifying originals - prediction_dfs = [df.copy() for df in prediction_dfs] - - # Print initial diagnostics - for i, df in enumerate(prediction_dfs): - print(f"\nDataFrame {i} initial state:") - print(f"Shape: {df.shape}") - print(f"Unique codes: {df['code'].nunique()}") - print(f"Date range: {df['date'].min()} to {df['date'].max()}") - print( - f"Forecast date range: {df['forecast_date'].min()} to {df['forecast_date'].max()}" - ) - print(f"Date dtype: {df['date'].dtype}") - print(f"Forecast date dtype: {df['forecast_date'].dtype}") - - # Ensure date columns are datetime - for df in prediction_dfs: - if not pd.api.types.is_datetime64_any_dtype(df["date"]): - df["date"] = pd.to_datetime(df["date"]) - if not pd.api.types.is_datetime64_any_dtype(df["forecast_date"]): - df["forecast_date"] = pd.to_datetime(df["forecast_date"]) - - # Drop duplicates in each dataframe - cleaned_dfs = [] - for i, df in enumerate(prediction_dfs): - cleaned_df = df.drop_duplicates( - subset=["forecast_date", "code", "date"], keep="last" - ) - print(f"\nAfter dropping duplicates, DataFrame {i}:") - print(f"Shape: {cleaned_df.shape}") - cleaned_dfs.append(cleaned_df) - - # Find common codes - common_codes = set(cleaned_dfs[0]["code"].unique()) - for df in cleaned_dfs[1:]: - common_codes = common_codes.intersection(set(df["code"].unique())) - print(f"\nNumber of common codes across all dataframes: {len(common_codes)}") - - # Filter for common codes - filtered_dfs = [] - for i, df in enumerate(cleaned_dfs): - filtered_df = df[df["code"].isin(common_codes)].copy() - print(f"\nAfter filtering for common codes, DataFrame {i}:") - print(f"Shape: {filtered_df.shape}") - filtered_dfs = cleaned_dfs # Keep all codes for now - - # Initialize result DataFrame - result_dfs = [] - - # Process each code separately - for code in common_codes: - code_dfs = [df[df["code"] == code] for df in filtered_dfs] - - common_forecast_dates = set(code_dfs[0]["forecast_date"]) - - for df in code_dfs[1:]: - common_forecast_dates = common_forecast_dates.intersection( - set(df["forecast_date"]) - ) - - # Filter for common dates - code_dfs = [ - df[df["forecast_date"].isin(common_forecast_dates)] for df in code_dfs - ] - - # Create result DataFrame for this code - if all(len(df) > 0 for df in code_dfs): - result_df = pd.DataFrame( - { - "code": code_dfs[0]["code"], - "forecast_date": code_dfs[0]["forecast_date"], - } - ) - - # Calculate means for aggregated columns - for col in columns_to_aggregate: - values = np.array([df[col].values for df in code_dfs]) - result_df[col] = np.nanmean(values, axis=0) - - result_dfs.append(result_df) - - # Combine all results - if result_dfs: - final_df = pd.concat(result_dfs, ignore_index=True) - print("\nFinal ensemble:") - print(f"Shape: {final_df.shape}") - print(f"Unique codes: {final_df['code'].nunique()}") - return final_df.sort_values(["code", "forecast_date"]).reset_index(drop=True) - else: - print("\nNo matching records found!") - return pd.DataFrame(columns=["code", "forecast_date"] + columns_to_aggregate) - - -def calculate_model_metrics( - obs_df, - model_predictions, - val_start_year, - val_end_year, - codes, - pentad_or_decad="pentad", -): - """ - Calculate metrics for multiple models against observations. - - Parameters: - ----------- - obs_df : pandas DataFrame - DataFrame containing the observations/linear regression predictions - model_predictions : dict - Dictionary of model predictions DataFrames with model names as keys - e.g., {'TFT': tft_df, 'TIDE': tide_df, ...} - val_start_year : int - Start year for validation period - val_end_year : int - End year for validation period - codes : list - List of station codes to process - - Returns: - -------- - pandas DataFrame - DataFrame containing calculated metrics for all models and stations - """ - metrics = pd.DataFrame() - - # iterate through the codes - for code in codes: - # Get observations for this code and filter date range - obs = obs_df[obs_df["code"] == code].copy() - obs = obs[obs["date"].dt.year.between(val_start_year, val_end_year)] - - metrics_list = [] - merged_df = obs.copy() - for model, df_org in model_predictions.items(): - df = df_org.copy() - df = df[df["code"] == code] - df["date"] = pd.to_datetime(df["forecast_date"]) - df[model] = df["Q50"] # Create model-specific column - # Select only needed columns for merge - df_to_merge = df[["date", "code", model]] - - # Merge with running merged_df - merged_df = pd.merge( - merged_df, df_to_merge, on=["date", "code"], how="inner" - ) - - # drop all rows with nan values - merged_df = merged_df.dropna() - - # Calculate metrics for linear regression first - if pentad_or_decad == "pentad": - metrics_linear = metric_functions.calculate_metrics_pentad( - merged_df, "forecasted_discharge", "discharge_avg", "delta" - ) - elif pentad_or_decad == "decad": - metrics_linear = metric_functions.calculate_metrics_decad( - merged_df, "forecasted_discharge", "discharge_avg", "delta" - ) - metrics_linear["model"] = "Linear Regression" - metrics_linear["code"] = code - metrics_list.append(metrics_linear) - - # Calculate metrics for each model - model_names = list(model_predictions.keys()) - for model_name in model_names: - # Calculate metrics - if pentad_or_decad == "pentad": - model_metrics = metric_functions.calculate_metrics_pentad( - merged_df, model_name, "discharge_avg", "delta" - ) - elif pentad_or_decad == "decad": - model_metrics = metric_functions.calculate_metrics_decad( - merged_df, model_name, "discharge_avg", "delta" - ) - else: - raise ValueError("Invalid value for 'pentad_or_decad'") - - model_metrics["model"] = model_name - model_metrics["code"] = code - - metrics_list.append(model_metrics) - - # Combine all metrics for this code - metrics_now = pd.concat(metrics_list) - metrics = pd.concat([metrics, metrics_now]) - - return metrics - - -def evaluate_metrics(): - save_dir = "../results_neural_discharge/Evaluation" - - val_start_year = int( - input("Enter the start year for validation period (eg. 2017): ") - ) - val_end_year = int(input("Enter the end year for validation period: ")) - - predictions_path = r"../../ml_operational_forecast/sensitive_data_forecast_tools/intermediate_data/predictions" - intermediate_data_path = ( - r"../../ml_operational_forecast/sensitive_data_forecast_tools/intermediate_data" - ) - - path_rr_mamba = "../Exogene_Models/Results/regional_KGZ/RRMamba_P_T_hindcast/hindcast_results.csv" - path_rr_lstm = "../Exogene_Models/Results/regional_KGZ/RRLSTM_P_T_hindcast/hindcast_results.csv" - - add_paths = [path_rr_mamba, path_rr_lstm] - - # models = ["Mamba", "Mamba_Assim", "LSTM", "LSTM_Assim",'TFT', 'TIDE', 'TSMIXER'] - models = ["TFT", "TIDE", "TSMIXER"] - # models = ['Mamba', 'LSTM'] - - processed_data_pentad = {} - processed_data_decad = {} - # Initialize these outside the loop - hydromet_discharge = None - linear_regression_pentad = None - linear_regression_decad = None - - for model in models: - print(f"Processing model {model}") - - if model == "Mamba": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, intermediate_data_path, model, add_path=add_paths[0] - ) - elif model == "Mamba_Assim": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, - intermediate_data_path, - model, - eval_on_assimilated=True, - add_path=add_paths[0], - ) - elif model == "LSTM": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, intermediate_data_path, model, add_path=add_paths[1] - ) - elif model == "LSTM_Assim": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, - intermediate_data_path, - model, - eval_on_assimilated=True, - add_path=add_paths[1], - ) - else: - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, intermediate_data_path, model - ) - - # save pentad to save_dir/timeseries - timeseries_dir = os.path.join(save_dir, "timeseries") - if not os.path.exists(timeseries_dir): - os.makedirs(timeseries_dir) - - pentad.to_csv( - os.path.join( - timeseries_dir, f"{model}_pentad_{val_start_year}_{val_end_year}.csv" - ), - index=False, - ) - decad.to_csv( - os.path.join( - timeseries_dir, f"{model}_decad_{val_start_year}_{val_end_year}.csv" - ), - index=False, - ) - - processed_data_pentad[model] = pentad.copy() - processed_data_decad[model] = decad.copy() - - # Only assign these once (from the first iteration) - if hydromet_discharge is None: - hydromet_discharge = hd - linear_regression_pentad = lr_p - linear_regression_decad = lr_d - - # Create ensembles for both pentad and decad - deep_ar_ens = ["TFT", "TIDE", "TSMIXER"] - # Create pentad ensemble - pentad_dfs = [processed_data_pentad[model].copy() for model in deep_ar_ens] - # discharge_cols = [col for col in pentad_dfs[0].columns if 'Q' in col] - discharge_cols = ["Q50"] - ensemble_pentad = create_ensemble(pentad_dfs, discharge_cols) - - # Create decad ensemble - # Debug decad data - decad_dfs = [processed_data_decad[model].copy() for model in deep_ar_ens] - discharge_cols_decad = ["Q50"] - ensemble_decad = create_ensemble(decad_dfs, discharge_cols_decad) - - print(len(ensemble_pentad)) - print("num unique codes in ensemble pentad: ", ensemble_pentad["code"].nunique()) - - processed_data_pentad["Neural Ensemble"] = ensemble_pentad.copy() - processed_data_decad["Neural Ensemble"] = ensemble_decad.copy() - """ - combined_ens = ['TFT', 'TIDE', 'TSMIXER', 'Mamba_Assim', 'LSTM_Assim'] - - # Calculate metrics for pentad - pentad_dfs = [processed_data_pentad[model].copy() for model in combined_ens] - discharge_cols = ['Q50'] - ensemble_pentad = create_ensemble(pentad_dfs, discharge_cols) - processed_data_pentad['combined_ensemble'] = ensemble_pentad.copy() - - # Calculate metrics for decad - decad_dfs = [processed_data_decad[model].copy() for model in combined_ens] - discharge_cols_decad = ['Q50'] - ensemble_decad = create_ensemble(decad_dfs, discharge_cols_decad) - processed_data_decad['combined_ensemble'] = ensemble_decad.copy() - - ex_ens = ['Mamba', 'LSTM'] - # Calculate metrics for pentad - pentad_dfs = [processed_data_pentad[model].copy() for model in ex_ens] - discharge_cols = ['Q50'] - ensemble_pentad = create_ensemble(pentad_dfs, discharge_cols) - processed_data_pentad['ex_ensemble'] = ensemble_pentad.copy() - - # Calculate metrics for decad - decad_dfs = [processed_data_decad[model].copy() for model in ex_ens] - discharge_cols_decad = ['Q50'] - ensemble_decad = create_ensemble(decad_dfs, discharge_cols_decad) - processed_data_decad['ex_ensemble'] = ensemble_decad.copy() - """ - - # Calculate metrics for pentad - metrics_pentad = calculate_model_metrics( - obs_df=linear_regression_pentad, - model_predictions=processed_data_pentad, - val_start_year=val_start_year, - val_end_year=val_end_year, - codes=linear_regression_pentad["code"].unique(), - pentad_or_decad="pentad", - ) - - # Calculate metrics for decad - metrics_decad = calculate_model_metrics( - obs_df=linear_regression_decad, - model_predictions=processed_data_decad, - val_start_year=val_start_year, - val_end_year=val_end_year, - codes=linear_regression_decad["code"].unique(), - pentad_or_decad="decad", - ) - - print(metrics_pentad.groupby("model").describe()) - # Save metrics to CSV - metrics_pentad.to_csv( - os.path.join( - save_dir, f"metrics/metrics_pentad_{val_start_year}_{val_end_year}.csv" - ), - index=False, - ) - metrics_decad.to_csv( - os.path.join( - save_dir, f"metrics/metrics_decad_{val_start_year}_{val_end_year}.csv" - ), - index=False, - ) - - -def get_daily_time_series(): - save_dir = "../results_neural_discharge/Evaluation" - - val_start_year = int( - input("Enter the start year for validation period (eg. 2017): ") - ) - val_end_year = int(input("Enter the end year for validation period: ")) - - predictions_path = r"../../ml_operational_forecast/sensitive_data_forecast_tools/intermediate_data/predictions" - intermediate_data_path = ( - r"../../ml_operational_forecast/sensitive_data_forecast_tools/intermediate_data" - ) - - path_rr_mamba = "../Exogene_Models/Results/regional_KGZ/RRMamba_P_T_hindcast/hindcast_results.csv" - path_rr_lstm = "../Exogene_Models/Results/regional_KGZ/RRLSTM_P_T_hindcast/hindcast_results.csv" - - add_paths = [path_rr_mamba, path_rr_lstm] - - # models = ["Mamba", "Mamba_Assim", "LSTM", "LSTM_Assim",'TFT', 'TIDE', 'TSMIXER'] - # models = ['TFT', 'Mamba_Assim'] - models = ["TFT", "TIDE", "TSMIXER"] - processed_data_pentad = {} - processed_data_decad = {} - # Initialize these outside the loop - hydromet_discharge = None - linear_regression_pentad = None - linear_regression_decad = None - - pentad_metrics_df = pd.DataFrame(columns=["model", "code", "NSE", "nRMSE", "CRPS"]) - decad_metrics_df = pd.DataFrame(columns=["model", "code", "NSE", "nRMSE", "CRPS"]) - - for model in models: - print(f"Processing model {model}") - - if model == "Mamba": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, - intermediate_data_path, - model, - add_path=add_paths[0], - return_daily_data=True, - ) - elif model == "Mamba_Assim": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, - intermediate_data_path, - model, - eval_on_assimilated=True, - add_path=add_paths[0], - return_daily_data=True, - ) - elif model == "LSTM": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, - intermediate_data_path, - model, - add_path=add_paths[1], - return_daily_data=True, - ) - elif model == "LSTM_Assim": - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, - intermediate_data_path, - model, - eval_on_assimilated=True, - add_path=add_paths[1], - return_daily_data=True, - ) - else: - pentad, decad, hd, lr_p, lr_d = process_model_data( - predictions_path, intermediate_data_path, model, return_daily_data=True - ) - - # round the daily data to 2 decimal places - pentad = pentad.round(2) - decad = decad.round(2) - - results_model_pentad = {} - results_model_decad = {} - - print( - "Len uniqe forecast dates pentad: ", len(pentad["forecast_date"].unique()) - ) - print("Len unique dates pentad: ", len(pentad["date"].unique())) - - # remove duplicates on subset of code and date - hd = hd.drop_duplicates(subset=["code", "date"], keep="last") - hd = hd.sort_values(by="date") - - pentad = pentad.drop_duplicates(subset=["code", "date"], keep="last") - pentad = pentad.sort_values(by="date") - - decad = decad.drop_duplicates(subset=["code", "date"], keep="last") - decad = decad.sort_values(by="date") - - for code in hd["code"].unique(): - # Get observations for this code and filter date range - obs = hd[hd["code"] == code].copy() - obs = obs[obs["date"].dt.year.between(val_start_year, val_end_year)] - - pentad_code = pentad[pentad["code"] == code].copy() - decad_code = decad[decad["code"] == code].copy() - - # everything to dt format - pentad_code["date"] = pd.to_datetime(pentad_code["date"]) - decad_code["date"] = pd.to_datetime(decad_code["date"]) - obs["date"] = pd.to_datetime(obs["date"]) - - # assert if length of any of the dataframes is zero - assert len(pentad_code) > 0, ( - "Length of pentad dataframe is zero - after code filter" - ) - assert len(decad_code) > 0, ( - "Length of decad dataframe is zero - after code filter" - ) - assert len(obs) > 0, ( - "Length of observation dataframe is zero - after code filter" - ) - - # Get the set of dates from each dataframe - obs_dates = set(obs["date"]) - pentad_dates = set(pentad_code["date"]) - decad_dates = set(decad_code["date"]) - - # Find common dates across all three dataframes - common_dates_pentad = obs_dates.intersection(pentad_dates).intersection( - obs_dates - ) - common_dates_decad = obs_dates.intersection(decad_dates).intersection( - obs_dates - ) - - # Filter the dataframes for common dates - obs_pentad = obs[obs["date"].isin(common_dates_pentad)].copy() - obs_decad = obs[obs["date"].isin(common_dates_decad)].copy() - pentad_code = pentad_code[ - pentad_code["date"].isin(common_dates_pentad) - ].copy() - decad_code = decad_code[decad_code["date"].isin(common_dates_decad)].copy() - - # assert if length of any of the dataframes is zero - assert len(pentad_code) > 0, ( - "Length of pentad dataframe is zero - after date filter" - ) - assert len(decad_code) > 0, ( - "Length of decad dataframe is zero - after date filter" - ) - assert len(obs_pentad) > 0, ( - "Length of pentad observation dataframe is zero - after date filter" - ) - assert len(obs_decad) > 0, ( - "Length of decad observation dataframe is zero - after date filter" - ) - - # if obs_pentad only contains nan values -> fill results with nan - if obs_pentad["discharge"].isnull().all(): - print(f"Obs pentad contains only nan values for code {code}") - nse_pentad = np.nan - nrmse_pentad = np.nan - crps_pentad = np.nan - - results_model_pentad[code] = [ - nse_pentad, - nrmse_pentad, - crps_pentad, - code, - ] - - if obs_decad["discharge"].isnull().all(): - print(f"Obs decad contains only nan values for code {code}") - nse_decad = np.nan - nrmse_decad = np.nan - crps_decad = np.nan - - results_model_decad[code] = [nse_decad, nrmse_decad, crps_decad, code] - continue - - # Calc NSE - nse_pentad = metric_functions.calculate_NSE( - obs_pentad["discharge"], pentad_code["Q50"] - ) - nse_decad = metric_functions.calculate_NSE( - obs_decad["discharge"], decad_code["Q50"] - ) - - # Calc nRMSE - nrmse_pentad = metric_functions.calculate_RMSE( - obs_pentad["discharge"], pentad_code["Q50"] - ) - nrmse_decad = metric_functions.calculate_RMSE( - obs_decad["discharge"], decad_code["Q50"] - ) - - # Calculate the CRPS - cols_with_Q = [col for col in pentad_code.columns if "Q" in col] - quantile_levels = [float(col[1:]) / 100 for col in cols_with_Q] - crps_pentad = np.nan - crps_decad = np.nan - # crps_pentad = metric_functions.calculate_mean_CRPS(obs_pentad['discharge'].values, pentad_code[cols_with_Q].values, quantile_levels) - # crps_decad = metric_functions.calculate_mean_CRPS(obs_decad['discharge'].values, decad_code[cols_with_Q].values, quantile_levels) - - results_model_pentad[code] = [nse_pentad, nrmse_pentad, crps_pentad, code] - results_model_decad[code] = [nse_decad, nrmse_decad, crps_decad, code] - - model_results_pentad = pd.DataFrame(results_model_pentad).T - model_results_pentad.columns = ["NSE", "nRMSE", "CRPS", "code"] - model_results_pentad["model"] = model - - model_results_decad = pd.DataFrame(results_model_decad).T - model_results_decad.columns = ["NSE", "nRMSE", "CRPS", "code"] - model_results_decad["model"] = model - - print(model_results_pentad.describe()) - - pentad_metrics_df = pd.concat([pentad_metrics_df, model_results_pentad]) - decad_metrics_df = pd.concat([decad_metrics_df, model_results_decad]) - - # save the daily time series - timeseries_dir = os.path.join(save_dir, "timeseries") - pentad.to_csv( - os.path.join( - timeseries_dir, - f"{model}_pentad_daily_{val_start_year}_{val_end_year}.csv", - ), - index=False, - ) - decad.to_csv( - os.path.join( - timeseries_dir, - f"{model}_decad_daily_{val_start_year}_{val_end_year}.csv", - ), - index=False, - ) - - # Save the metrics - pentad_metrics_df.to_csv( - os.path.join( - save_dir, f"daily_metrics_pentad_daily_{val_start_year}_{val_end_year}.csv" - ), - index=False, - ) - decad_metrics_df.to_csv( - os.path.join( - save_dir, f"daily_metrics_decad_daily_{val_start_year}_{val_end_year}.csv" - ), - index=False, - ) - - -if __name__ == "__main__": - eval_metrics_or_daily = input( - "Do you want to retrive the metrics or daily time series (0/1): " - ) - - if eval_metrics_or_daily == "0": - evaluate_metrics() - - elif eval_metrics_or_daily == "1": - get_daily_time_series() diff --git a/st_forecast/evaluation/sapphire_metrics.py b/st_forecast/evaluation/sapphire_metrics.py deleted file mode 100644 index 9039c2b..0000000 --- a/st_forecast/evaluation/sapphire_metrics.py +++ /dev/null @@ -1,1252 +0,0 @@ -import os -import pandas as pd -import numpy as np -import re -import math -import datetime as dt -import logging - -# Configure logging -logger = logging.getLogger(__name__) -# Set the logging level to DEBUG for detailed output -logger.setLevel(logging.DEBUG) - -# Create a console handler and set its level to DEBUG -console_handler = logging.StreamHandler() -console_handler.setLevel(logging.DEBUG) - -# Create a formatter and add it to the handler -formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") -console_handler.setFormatter(formatter) - -# Add the handler to the logger -logger.addHandler(console_handler) - -# Import custom modules - adjust these paths as needed -try: - import tl # time-related utilities -except ImportError: - logger.warning( - "Could not import 'tl' module. Some time-related functions may not work." - ) - tl = None - -global EVAL_START -EVAL_START = 2010 - - -def sdivsigma_nse(data: pd.DataFrame, observed_col: str, simulated_col: str): - """ - Calculate the forecast efficacy and the Nash-Sutcliffe Efficiency (NSE) for the observed and simulated data. - - NSE = 1 - s/sigma - - Args: - data (pandas.DataFrame): The input data containing the observed and simulated data. - observed_col (str): The name of the column containing the observed data. - simulated_col (str): The name of the column containing the simulated data. - - Returns: - pandas.Series: A pandas Series containing the forecast efficacy and the NSE value. - - Raises: - ValueError: If the input data is missing one or more required columns. - - """ - # Test the input. Make sure that the DataFrame contains the required columns - if not all(column in data.columns for column in [observed_col, simulated_col]): - raise ValueError( - f"DataFrame is missing one or more required columns: {observed_col, simulated_col}" - ) - - # print("DEBUG: forecasting:sdivsigma_nse: data: \n", data) - - # Convert to numpy arrays for faster computation - # Use float64 for better numerical stability - observed = data[observed_col].to_numpy(dtype=np.float64) - simulated = data[simulated_col].to_numpy(dtype=np.float64) - - # Check for empty data after dropping NaNs - mask = ~(np.isnan(observed) | np.isnan(simulated)) - if not np.any(mask): - return pd.Series([np.nan, np.nan], index=["sdivsigma", "nse"]) - - # Filter arrays using mask - observed = observed[mask] - simulated = simulated[mask] - - # Early return if not enough data points - if len(observed) < 2: # Need at least 2 points for std calculation - return pd.Series([np.nan, np.nan], index=["sdivsigma", "nse"]) - - # Calculate mean once for reuse - observed_mean = np.mean(observed) - - # Count the number of data points - n = len(observed) - - # Calculate denominators - denominator_nse = np.sum((observed - observed_mean) ** 2) - # sigma: Standard deviation of the observed data - denominator_sdivsigma = np.std(observed, ddof=1) # ddof=1 for sample std - - # Check for numerical stability - if denominator_nse < 1e-10 or denominator_sdivsigma < 1e-10: - return pd.Series([np.nan, np.nan], index=["sdivsigma", "nse"]) - - try: - # Calculate differences once for reuse - differences = observed - simulated - - # Calculate NSE - numerator_nse = np.sum(differences**2) - nse_value = 1 - (numerator_nse / denominator_nse) - - # Calculate sdivsigma - # s: Average of squared differences between observed and simulated data - numerator_sdivsigma = np.sqrt(np.sum(differences**2) / (n - 1)) - # s/sigma: Efficacy of the model - sdivsigma = numerator_sdivsigma / denominator_sdivsigma - - # Sanity checks - if not (-np.inf < nse_value < np.inf) or not (0 <= sdivsigma < np.inf): - return pd.Series([np.nan, np.nan], index=["sdivsigma", "nse"]) - - return pd.Series([sdivsigma, nse_value], index=["sdivsigma", "nse"]) - - except (RuntimeWarning, FloatingPointError): - return pd.Series([np.nan, np.nan], index=["sdivsigma", "nse"]) - - -def forecast_accuracy_hydromet( - data: pd.DataFrame, observed_col: str, simulated_col: str, delta_col: str -): - """ - Calculate the forecast accuracy for the observed and simulated data. - - Args: - data (pandas.DataFrame): The input data containing the observed and simulated data. - observed_col (str): The name of the column containing the observed data. - simulated_col (str): The name of the column containing the simulated data. - - Returns: - pandas.Series: A pandas Series containing the forecast accuracy. - - Raises: - ValueError: If the input data is missing one or more required columns. - - """ - # Test the input. Make sure that the DataFrame contains the required columns - if not all( - column in data.columns for column in [observed_col, simulated_col, delta_col] - ): - raise ValueError( - f"DataFrame is missing one or more required columns: {observed_col, simulated_col, delta_col}" - ) - - # Convert to numpy arrays for faster computation - observed = data[observed_col].to_numpy(dtype=np.float64) - simulated = data[simulated_col].to_numpy(dtype=np.float64) - delta_values = data[delta_col].to_numpy(dtype=np.float64) - - # Check for empty data after dropping NaNs - mask = ~(np.isnan(observed) | np.isnan(simulated) | np.isnan(delta_values)) - if not np.any(mask): - return pd.Series([np.nan, np.nan], index=["delta", "accuracy"]) - - # Also drop rows where observed, simulated or delta_valus is inf - mask = mask & ~(np.isinf(observed) | np.isinf(simulated) | np.isinf(delta_values)) - if not np.any(mask): - return pd.Series([np.nan, np.nan], index=["delta", "accuracy"]) - - # Filter arrays using mask - observed = observed[mask] - simulated = simulated[mask] - delta_values = delta_values[mask] - - # Early return if not enough data points - if len(observed) < 1: - return pd.Series([np.nan, np.nan], index=["delta", "accuracy"]) - - try: - # Calculate absolute differences once - abs_diff = np.abs(observed - simulated) - - # Calculate accuracy using vectorized operations - accuracy = np.mean(abs_diff <= delta_values) - - # Get the last delta value (they are all the same) - delta = delta_values[-1] - - # Sanity checks - if not (0 <= accuracy <= 1) or not (0 <= delta < np.inf): - return pd.Series([np.nan, np.nan], index=["delta", "accuracy"]) - - return pd.Series([delta, accuracy], index=["delta", "accuracy"]) - - except (RuntimeWarning, FloatingPointError): - return pd.Series([np.nan, np.nan], index=["delta", "accuracy"]) - - -def mae(data: pd.DataFrame, observed_col: str, simulated_col: str): - """ - Calculate the mean average error between observed and simulated data - - Args: - data (pandas.DataFrame): The input data containing the observed and simulated data. - observed_col (str): The name of the column containing the observed data. - simulated_col (str): The name of the column containing the simulated data. - - Returns: - pandas.Series: A series containing: - - mae: mean average error between observed and simulated data - - n_pairs: number of valid observed-simulated pairs used in calculation - - Raises: - ValueError: If the input data is missing one or more required columns. - """ - # Test the input. Make sure that the DataFrame contains the required columns - if not all(column in data.columns for column in [observed_col, simulated_col]): - raise ValueError( - f"DataFrame is missing one or more required columns: {observed_col, simulated_col}" - ) - - # Convert to numpy arrays for faster computation - observed = data[observed_col].to_numpy(dtype=np.float64) - simulated = data[simulated_col].to_numpy(dtype=np.float64) - - # Check for empty data after dropping NaNs - mask = ~(np.isnan(observed) | np.isnan(simulated)) - if not np.any(mask): - return pd.Series([np.nan, 0], index=["mae", "n_pairs"]) - - # Filter arrays using mask - observed = observed[mask] - simulated = simulated[mask] - - # Early return if not enough data points - if len(observed) < 1: - return pd.Series([np.nan, 0], index=["mae", "n_pairs"]) - - try: - # Calculate MAE using vectorized operations - mae_value = np.mean(np.abs(observed - simulated)) - - # Sanity check - if not (0 <= mae_value < np.inf): # MAE must be non-negative - return pd.Series([np.nan, 0], index=["mae", "n_pairs"]) - - return pd.Series([mae_value, len(observed)], index=["mae", "n_pairs"]) - - except (RuntimeWarning, FloatingPointError): - return pd.Series([np.nan, 0], index=["mae", "n_pairs"]) - - -def calculate_forecast_skill_deprecating( - data_df: pd.DataFrame, - station_col: str, - pentad_col: str, - observation_col: str, - simulation_col: str, -) -> pd.DataFrame: - """ - Calculates the forecast skill for each group in the input data. - - Args: - data (pandas.DataFrame): The input data containing the observation and - simulation data. - station_col (str): The name of the column containing the station - identifier. - pentad_col (str): The name of the column containing the pentad data. - observation_col (str): The name of the column containing the observation - data. - simulation_col (str): The name of the column containing the simulation - data. - - Returns: - pandas.DataFrame: The modified input data with additional columns - containing the forecast skill information, namely abolute error, - observation_std0674 and flag. - """ - - # Test the input. Make sure that the DataFrame contains the required columns - if not all( - column in data_df.columns - for column in [station_col, pentad_col, observation_col, simulation_col] - ): - raise ValueError( - f"DataFrame is missing one or more required columns: {station_col, pentad_col, observation_col, simulation_col}" - ) - - # Initialize columns - data_df.loc[:, "absolute_error"] = 0.0 - data_df.loc[:, "observation_std0674"] = 0.0 # delta - data_df.loc[:, "flag"] = 0.0 - data_df.loc[:, "accuracy"] = 0.0 # percentage of good forecasts - data_df.loc[:, "observation_std"] = 0.0 # sigma - data_df.loc[:, "observation_std_sanity_check"] = 0.0 # sigma - data_df.loc[:, "forecast_std"] = 0.0 # s - data_df.loc[:, "sdivsigma"] = 0.0 # s / sigma, "effectiveness" of the model - - # Loop over each station and pentad - for station in data_df[station_col].unique(): - for pentad in data_df[pentad_col].unique(): - # Get the data for the station and pentad - station_data = data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), : - ] - - # Drop NaN values - station_data = station_data.dropna() - - # Calculate the absolute error between the simulation data and the observation data - absolute_error_forecast = abs( - station_data[simulation_col] - station_data[observation_col] - ) - absolute_error_observed = abs( - station_data[observation_col] - station_data[observation_col].mean() - ) - observation_std = station_data[observation_col].std() - # The unbiased sample standard deviation sigma is calculated as - # sigma = sqrt(sum((x_i - x_mean)^2) / (n-1)) - observation_std_sanity_check = math.sqrt( - station_data[observation_col] - .apply(lambda x: (x - station_data[observation_col].mean()) ** 2) - .sum() - / (len(station_data) - 1) - ) - # The measure s is calculated as - # s = sqrt(sum((x_i - y_i)^2) / (n-2)) - forecast_std = math.sqrt( - station_data.apply( - lambda x: (x[observation_col] - x[simulation_col]) ** 2, axis=1 - ).sum() - / (len(station_data) - 2) - ) - sdivsigma = forecast_std / observation_std - - # Note: .std() will yield NaN if there is only one value in the DataFrame - # Test if the standard deviation is NaN and return 0.0 if it is - if np.isnan(observation_std): - observation_std = 0.0 - - # Set the flag if the error is smaller than 0.674 times the standard deviation of the observation data - flag = absolute_error_forecast <= 0.674 * observation_std - - # Calculate the accuracy of the forecast - accuracy = flag.mean() - - # Delta is 0.674 times the standard deviation of the observation data - # This is the measure for the allowable uncertainty of a good forecast - observation_std0674 = 0.674 * observation_std - - # Store the slope and intercept in the data_df - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "absolute_error", - ] = absolute_error_forecast - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "observation_std0674", - ] = observation_std0674 - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "flag", - ] = flag - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "observation_std", - ] = observation_std - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "observation_std_sanity_check", - ] = observation_std_sanity_check - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "forecast_std", - ] = forecast_std - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "sdivsigma", - ] = sdivsigma - data_df.loc[ - (data_df[station_col] == station) & (data_df[pentad_col] == pentad), - "accuracy", - ] = accuracy - - # print("DEBUG: fl.calculate_forecast_skill: data_df\n", data_df.head(20)) - # print(data_df.tail(20)) - - return data_df - - -def calculate_skill_metrics_pentad( - observed: pd.DataFrame, simulated: pd.DataFrame, timing_stats=None -): - """ - For each model and hydropost in the simulated DataFrame, calculates a number - of skill metrics based on the observed DataFrame. - - Args: - observed (pd.DataFrame): The DataFrame containing the observed data. - simulated (pd.DataFrame): The DataFrame containing the simulated data. - timing_stats (TimingStats, optional): Timing statistics collector (ignored for simplicity) - - Returns: - pd.DataFrame: The DataFrame containing the skill metrics for each model - and hydropost. - pd.DataFrame: Combined forecasts and observations DataFrame - timing_stats: Timing statistics collector (returned as passed) - """ - - # Test the input. Make sure that the DataFrames contain the required columns - if not all( - column in observed.columns - for column in [ - "code", - "date", - "discharge_avg", - "model_long", - "model_short", - "delta", - ] - ): - raise ValueError( - f"Observed DataFrame is missing one or more required columns: {['code', 'date', 'discharge_avg', 'model_long', 'model_short', 'delta']}" - ) - if not all( - column in simulated.columns - for column in [ - "code", - "date", - "pentad_in_year", - "forecasted_discharge", - "model_long", - "model_short", - ] - ): - raise ValueError( - f"Simulated DataFrame is missing one or more required columns: {['code', 'date', 'pentad_in_year', 'forecasted_discharge', 'model_long', 'model_short']}" - ) - - # Check if 'pentad' column exists and rename it to 'pentad_in_month' if needed - if "pentad" in simulated.columns and "pentad_in_month" not in simulated.columns: - simulated["pentad_in_month"] = simulated["pentad"] - logger.debug( - "Renamed 'pentad' column to 'pentad_in_month' in simulated DataFrame" - ) - - # Local functions - def test_for_tuples(df): - # Identify tuples in each cell - is_tuple = df.apply(lambda col: col.map(lambda x: isinstance(x, tuple))) - # Check if there are any True values in is_tuple - contains_tuples = is_tuple.any(axis=1).any() - # Test if there are any tuples in the DataFrame - if contains_tuples: - logger.debug("There are tuples after the merge.") - - # Step 2: Filter rows that contain any tuples - rows_with_tuples = df[is_tuple.any(axis=1)] - - # Print rows with tuples - logger.debug(rows_with_tuples) - else: - logger.debug("No tuples found after the merge.") - - def extract_first_parentheses_content(string_list): - pattern = r"\((.*?)\)" - - result = [] - for string in string_list: - match = re.search(pattern, string) - if match: - result.append(match.group(1)) - else: - result.append("") # or None, or any other placeholder - - return result - - def model_long_agg(x): - # Get unique models - model_list = x.unique() - # Only keep strings within brackets (), discard the rest of the string and the brackets - short_model_list = extract_first_parentheses_content(model_list) - # Concatenat the model names - unique_models = ", ".join(sorted(short_model_list)) - return f"Ens. Mean with {unique_models} (EM)" - - def model_short_agg(x): - return "EM" - - def filter_for_highly_skilled_forecasts(skill_stats): - """ - Filter the skill_stats DataFrame for highly skilled forecasts based on predefined thresholds. - """ - # Use hardcoded thresholds instead of environment variables - threshold_sdivsigma = 0.6 - threshold_accuracy = 0.8 - threshold_nse = 0.8 - - # Filter for rows where sdivsigma is smaller than the threshold - skill_stats_ensemble = skill_stats[ - skill_stats["sdivsigma"] < threshold_sdivsigma - ].copy() - - # Filter for rows where accuracy is larger than the threshold - skill_stats_ensemble = skill_stats_ensemble[ - skill_stats_ensemble["accuracy"] > threshold_accuracy - ].copy() - - # Filter for rows where nse is larger than the threshold - skill_stats_ensemble = skill_stats_ensemble[ - skill_stats_ensemble["nse"] > threshold_nse - ].copy() - - return skill_stats_ensemble - - # Filter data - We calculate skill metrics only on forecasts after 2010 - observed = observed[observed["date"].dt.year >= EVAL_START] - simulated = simulated[simulated["date"].dt.year >= EVAL_START] - - # Merge the observed and simulated DataFrames - skill_metrics_df = pd.merge( - simulated, - observed[["code", "date", "discharge_avg", "delta"]], - on=["code", "date"], - ) - test_for_tuples(skill_metrics_df) - - # Calculate the skill metrics for each group based on the 'pentad_in_year', 'code' and 'model' columns - skill_stats = ( - skill_metrics_df.groupby( - ["pentad_in_year", "code", "model_long", "model_short"] - )[skill_metrics_df.columns] - .apply( - sdivsigma_nse, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - ) - .reset_index() - ) - test_for_tuples(skill_stats) - - # Calculate MAE stats - mae_stats = ( - skill_metrics_df.groupby( - ["pentad_in_year", "code", "model_long", "model_short"] - )[skill_metrics_df.columns] - .apply(mae, observed_col="discharge_avg", simulated_col="forecasted_discharge") - .reset_index() - ) - test_for_tuples(mae_stats) - - # Calculate accuracy stats - accuracy_stats = ( - skill_metrics_df.groupby( - ["pentad_in_year", "code", "model_long", "model_short"] - )[skill_metrics_df.columns] - .apply( - forecast_accuracy_hydromet, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - delta_col="delta", - ) - .reset_index() - ) - test_for_tuples(accuracy_stats) - - # Merge all skill stats - skill_stats = pd.merge( - skill_stats, - accuracy_stats, - on=["pentad_in_year", "code", "model_long", "model_short"], - ) - test_for_tuples(skill_stats) - - skill_stats = pd.merge( - skill_stats, - mae_stats, - on=["pentad_in_year", "code", "model_long", "model_short"], - ) - test_for_tuples(skill_stats) - - # Calculate ensemble skill metrics for highly skilled forecasts - # Calculate ensemble skill metrics for highly skilled forecasts - skill_stats_ensemble = filter_for_highly_skilled_forecasts(skill_stats) - - # Now we get the rows from the skill_metrics_df where pentad_in_year, code, - # model_long and model_short are the same as in skill_stats_ensemble - skill_metrics_df_ensemble = skill_metrics_df[ - skill_metrics_df["pentad_in_year"].isin(skill_stats_ensemble["pentad_in_year"]) - & skill_metrics_df["code"].isin(skill_stats_ensemble["code"]) - & skill_metrics_df["model_long"].isin(skill_stats_ensemble["model_long"]) - & skill_metrics_df["model_short"].isin(skill_stats_ensemble["model_short"]) - ].copy() - # Filter out rows where forecasted_discharge is NaN - skill_metrics_df_ensemble = skill_metrics_df_ensemble.dropna( - subset=["forecasted_discharge"] - ).copy() - - # Drop columns with model_short == NE (neural ensemble) - skill_metrics_df_ensemble = skill_metrics_df_ensemble[ - skill_metrics_df_ensemble["model_short"] != "NE" - ].copy() - - # Perform the aggregations and keep only the unique combinations - skill_metrics_df_ensemble_avg = ( - skill_metrics_df_ensemble.groupby(["date", "code"]) - .agg( - { - "pentad_in_year": "first", - "forecasted_discharge": "mean", - "model_long": model_long_agg, - "model_short": model_short_agg, - } - ) - .reset_index() - ) - - # Discard rows with model_long equal to 'Ensemble Mean with (EM)' or equal to Ensemble Mean with LR (EM) - skill_metrics_df_ensemble_avg = skill_metrics_df_ensemble_avg[ - (skill_metrics_df_ensemble_avg["model_long"] != "Ens. Mean with (EM)") - & (skill_metrics_df_ensemble_avg["model_long"] != "Ens. Mean with LR (EM)") - ].copy() - - # Now recalculate the skill metrics for the ensemble - ensemble_skill_metrics_df = pd.merge( - skill_metrics_df_ensemble_avg, - observed[["code", "date", "discharge_avg", "delta"]], - on=["code", "date"], - ) - - number_of_models = simulated["model_long"].nunique() - - if number_of_models > 1: - ensemble_skill_stats = ( - ensemble_skill_metrics_df.groupby( - ["pentad_in_year", "code", "model_long", "model_short"] - )[ensemble_skill_metrics_df.columns] - .apply( - sdivsigma_nse, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - ) - .reset_index() - ) - - ensemble_mae_stats = ( - ensemble_skill_metrics_df.groupby( - ["pentad_in_year", "code", "model_long", "model_short"] - )[ensemble_skill_metrics_df.columns] - .apply( - mae, observed_col="discharge_avg", simulated_col="forecasted_discharge" - ) - .reset_index() - ) - - ensemble_accuracy_stats = ( - ensemble_skill_metrics_df.groupby( - ["pentad_in_year", "code", "model_long", "model_short"] - )[ensemble_skill_metrics_df.columns] - .apply( - forecast_accuracy_hydromet, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - delta_col="delta", - ) - .reset_index() - ) - - ensemble_skill_stats = pd.merge( - ensemble_skill_stats, - ensemble_mae_stats, - on=["pentad_in_year", "code", "model_long", "model_short"], - ) - ensemble_skill_stats = pd.merge( - ensemble_skill_stats, - ensemble_accuracy_stats, - on=["pentad_in_year", "code", "model_long", "model_short"], - ) - - # Append the ensemble skill metrics to the skill metrics - skill_stats = pd.concat([skill_stats, ensemble_skill_stats], ignore_index=True) - - # Add ensemble mean forecasts to simulated dataframe - # Calculate pentad in month (add 1 day to date) - if tl is not None: - ensemble_skill_metrics_df["pentad_in_month"] = ( - ensemble_skill_metrics_df["date"] + dt.timedelta(days=1.0) - ).apply(tl.get_pentad) - else: - logger.warning("tl module not available, setting pentad_in_month to NaN") - ensemble_skill_metrics_df["pentad_in_month"] = np.nan - - # Join the two dataframes - joint_forecasts = pd.merge( - simulated, - ensemble_skill_metrics_df[ - [ - "code", - "date", - "pentad_in_month", - "pentad_in_year", - "forecasted_discharge", - "model_long", - "model_short", - ] - ], - on=[ - "code", - "date", - "pentad_in_month", - "pentad_in_year", - "model_long", - "model_short", - "forecasted_discharge", - ], - how="outer", - ) - else: - joint_forecasts = simulated.copy() - - return skill_stats, joint_forecasts, timing_stats - - -def calculate_skill_metrics_decade( - observed: pd.DataFrame, simulated: pd.DataFrame, timing_stats=None -): - """ - For each model and hydropost in the simulated DataFrame, calculates a number - of skill metrics based on the observed DataFrame. - - Args: - observed (pd.DataFrame): The DataFrame containing the observed data. - simulated (pd.DataFrame): The DataFrame containing the simulated data. - timing_stats (TimingStats, optional): Timing statistics collector (ignored for simplicity) - - Returns: - pd.DataFrame: The DataFrame containing the skill metrics for each model - and hydropost. - pd.DataFrame: Combined forecasts and observations DataFrame - timing_stats: Timing statistics collector (returned as passed) - """ - - # Test the input. Make sure that the DataFrames contain the required columns - if not all( - column in observed.columns - for column in [ - "code", - "date", - "discharge_avg", - "model_long", - "model_short", - "delta", - ] - ): - raise ValueError( - f"Observed DataFrame is missing one or more required columns: {['code', 'date', 'discharge_avg', 'model_long', 'model_short', 'delta']}" - ) - if not all( - column in simulated.columns - for column in [ - "code", - "date", - "decad_in_year", - "forecasted_discharge", - "model_long", - "model_short", - ] - ): - raise ValueError( - f"Simulated DataFrame is missing one or more required columns: {['code', 'date', 'decad_in_year', 'forecasted_discharge', 'model_long', 'model_short']}" - ) - - # Check if 'decad' column exists and rename it to 'decad_in_month' if needed - if "decad" in simulated.columns and "decad_in_month" not in simulated.columns: - simulated["decad_in_month"] = simulated["decad"] - logger.debug( - "Renamed 'decad' column to 'decad_in_month' in simulated DataFrame" - ) - - # Print column names of simulated - logger.debug(f"DEBUG: simulated.columns\n{simulated.columns}") - - # Local functions - def test_for_tuples(df): - # Identify tuples in each cell - is_tuple = df.apply(lambda col: col.map(lambda x: isinstance(x, tuple))) - # Check if there are any True values in is_tuple - contains_tuples = is_tuple.any(axis=1).any() - # Test if there are any tuples in the DataFrame - if contains_tuples: - logger.debug("There are tuples after the merge.") - - # Step 2: Filter rows that contain any tuples - rows_with_tuples = df[is_tuple.any(axis=1)] - - # Print rows with tuples - logger.debug(rows_with_tuples) - else: - logger.debug("No tuples found after the merge.") - - def extract_first_parentheses_content(string_list): - pattern = r"\((.*?)\)" - - result = [] - for string in string_list: - match = re.search(pattern, string) - if match: - result.append(match.group(1)) - else: - result.append("") # or None, or any other placeholder - - return result - - def model_long_agg(x): - # Get unique models - model_list = x.unique() - # Only keep strings within brackets (), discard the rest of the string and the brackets - short_model_list = extract_first_parentheses_content(model_list) - # Concatenat the model names - unique_models = ", ".join(sorted(short_model_list)) - return f"Ens. Mean with {unique_models} (EM)" - - def model_short_agg(x): - return "EM" - - def filter_for_highly_skilled_forecasts(skill_stats): - """ - Filter the skill_stats DataFrame for highly skilled forecasts based on predefined thresholds. - """ - # Use hardcoded thresholds instead of environment variables - threshold_sdivsigma = 0.6 - threshold_accuracy = 0.8 - threshold_nse = 0.8 - - # Filter for rows where sdivsigma is smaller than the threshold - skill_stats_ensemble = skill_stats[ - skill_stats["sdivsigma"] < threshold_sdivsigma - ].copy() - - # Filter for rows where accuracy is larger than the threshold - skill_stats_ensemble = skill_stats_ensemble[ - skill_stats_ensemble["accuracy"] > threshold_accuracy - ].copy() - - # Filter for rows where nse is larger than the threshold - skill_stats_ensemble = skill_stats_ensemble[ - skill_stats_ensemble["nse"] > threshold_nse - ].copy() - - return skill_stats_ensemble - - # Filter data - We calculate skill metrics only on forecasts after 2010 - observed = observed[observed["date"].dt.year >= EVAL_START] - simulated = simulated[simulated["date"].dt.year >= EVAL_START] - - # Print column names of simulated - logger.debug(f"DEBUG: simulated.columns\n{simulated.columns}") - - # Merge the observed and simulated DataFrames - skill_metrics_df = pd.merge( - simulated, - observed[["code", "date", "discharge_avg", "delta"]], - on=["code", "date"], - ) - test_for_tuples(skill_metrics_df) - - # Calculate the skill metrics for each group based on the 'decad_in_year', 'code' and 'model' columns - skill_stats = ( - skill_metrics_df.groupby( - ["decad_in_year", "code", "model_long", "model_short"] - )[skill_metrics_df.columns] - .apply( - sdivsigma_nse, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - ) - .reset_index() - ) - test_for_tuples(skill_stats) - - # Calculate MAE stats - mae_stats = ( - skill_metrics_df.groupby( - ["decad_in_year", "code", "model_long", "model_short"] - )[skill_metrics_df.columns] - .apply(mae, observed_col="discharge_avg", simulated_col="forecasted_discharge") - .reset_index() - ) - test_for_tuples(mae_stats) - - # Calculate accuracy stats - accuracy_stats = ( - skill_metrics_df.groupby( - ["decad_in_year", "code", "model_long", "model_short"] - )[skill_metrics_df.columns] - .apply( - forecast_accuracy_hydromet, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - delta_col="delta", - ) - .reset_index() - ) - test_for_tuples(accuracy_stats) - - # Merge all skill stats - skill_stats = pd.merge( - skill_stats, - accuracy_stats, - on=["decad_in_year", "code", "model_long", "model_short"], - ) - test_for_tuples(skill_stats) - - skill_stats = pd.merge( - skill_stats, - mae_stats, - on=["decad_in_year", "code", "model_long", "model_short"], - ) - test_for_tuples(skill_stats) - - # Calculate ensemble skill metrics for highly skilled forecasts - # Calculate ensemble skill metrics for highly skilled forecasts - skill_stats_ensemble = filter_for_highly_skilled_forecasts(skill_stats) - - # Now we get the rows from the skill_metrics_df where decad_in_year, code, - # model_long and model_short are the same as in skill_stats_ensemble - skill_metrics_df_ensemble = skill_metrics_df[ - skill_metrics_df["decad_in_year"].isin(skill_stats_ensemble["decad_in_year"]) - & skill_metrics_df["code"].isin(skill_stats_ensemble["code"]) - & skill_metrics_df["model_long"].isin(skill_stats_ensemble["model_long"]) - & skill_metrics_df["model_short"].isin(skill_stats_ensemble["model_short"]) - ].copy() - - # Drop columns with model_short == NE (neural ensemble) - skill_metrics_df_ensemble = skill_metrics_df_ensemble[ - skill_metrics_df_ensemble["model_short"] != "NE" - ].copy() - - # Perform the aggregations and keep only the unique combinations - skill_metrics_df_ensemble_avg = ( - skill_metrics_df_ensemble.groupby(["date", "code"]) - .agg( - { - "decad_in_year": "first", - "forecasted_discharge": "mean", - "model_long": model_long_agg, - "model_short": model_short_agg, - } - ) - .reset_index() - ) - - # Discard rows with model_long equal to 'Ensemble Mean with (EM)' or equal to Ensemble Mean with LR (EM) - skill_metrics_df_ensemble_avg = skill_metrics_df_ensemble_avg[ - (skill_metrics_df_ensemble_avg["model_long"] != "Ens. Mean with (EM)") - & (skill_metrics_df_ensemble_avg["model_long"] != "Ens. Mean with LR (EM)") - ].copy() - - # Now recalculate the skill metrics for the ensemble - ensemble_skill_metrics_df = pd.merge( - skill_metrics_df_ensemble_avg, - observed[["code", "date", "discharge_avg", "delta"]], - on=["code", "date"], - ) - - number_of_models = simulated["model_long"].nunique() - - if number_of_models > 1: - ensemble_skill_stats = ( - ensemble_skill_metrics_df.groupby( - ["decad_in_year", "code", "model_long", "model_short"] - )[ensemble_skill_metrics_df.columns] - .apply( - sdivsigma_nse, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - ) - .reset_index() - ) - - ensemble_mae_stats = ( - ensemble_skill_metrics_df.groupby( - ["decad_in_year", "code", "model_long", "model_short"] - )[ensemble_skill_metrics_df.columns] - .apply( - mae, observed_col="discharge_avg", simulated_col="forecasted_discharge" - ) - .reset_index() - ) - - ensemble_accuracy_stats = ( - ensemble_skill_metrics_df.groupby( - ["decad_in_year", "code", "model_long", "model_short"] - )[ensemble_skill_metrics_df.columns] - .apply( - forecast_accuracy_hydromet, - observed_col="discharge_avg", - simulated_col="forecasted_discharge", - delta_col="delta", - ) - .reset_index() - ) - - ensemble_skill_stats = pd.merge( - ensemble_skill_stats, - ensemble_mae_stats, - on=["decad_in_year", "code", "model_long", "model_short"], - ) - ensemble_skill_stats = pd.merge( - ensemble_skill_stats, - ensemble_accuracy_stats, - on=["decad_in_year", "code", "model_long", "model_short"], - ) - - # Append the ensemble skill metrics to the skill metrics - skill_stats = pd.concat([skill_stats, ensemble_skill_stats], ignore_index=True) - - # Add ensemble mean forecasts to simulated dataframe - # Calculate decad in month (add 1 day to date) - if tl is not None: - ensemble_skill_metrics_df["decad_in_month"] = ( - ensemble_skill_metrics_df["date"] + dt.timedelta(days=1.0) - ).apply(tl.get_decad_in_month) - else: - logger.warning("tl module not available, setting decad_in_month to NaN") - ensemble_skill_metrics_df["decad_in_month"] = np.nan - - # Join the two dataframes - joint_forecasts = pd.merge( - simulated, - ensemble_skill_metrics_df[ - [ - "code", - "date", - "decad_in_month", - "decad_in_year", - "forecasted_discharge", - "model_long", - "model_short", - ] - ], - on=[ - "code", - "date", - "decad_in_month", - "decad_in_year", - "model_long", - "model_short", - "forecasted_discharge", - ], - how="outer", - ) - - else: - joint_forecasts = simulated.copy() - - return skill_stats, joint_forecasts, timing_stats - - -def read_obs_pred_tjk(): - obs_pentad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/taj_data_forecast_tools/intermediate_data/runoff_pentad.csv" - obs_decad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/taj_data_forecast_tools/intermediate_data/runoff_decad.csv" - - pred_pentad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/taj_data_forecast_tools/intermediate_data/combined_forecasts_pentad.csv" - pred_decad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/taj_data_forecast_tools/intermediate_data/combined_forecasts_decad.csv" - - # Read the observed data - observed_pentad = pd.read_csv(obs_pentad_path) - observed_decad = pd.read_csv(obs_decad_path) - - # Add a column model to the dataframe - observed_pentad["model_long"] = "Observed (Obs)" - observed_pentad["model_short"] = "Obs" - observed_decad["model_long"] = "Observed (Obs)" - observed_decad["model_short"] = "Obs" - - # Calculate delta (allowable uncertainty threshold) for each station and time period - # Delta = 0.674 * standard deviation of observed discharge for each code and pentad_in_year - observed_pentad["delta"] = observed_pentad.groupby(["code", "pentad_in_year"])[ - "discharge_avg" - ].transform(lambda x: 0.674 * x.std()) - observed_decad["delta"] = observed_decad.groupby(["code", "decad_in_year"])[ - "discharge_avg" - ].transform(lambda x: 0.674 * x.std()) - - # Fill NaN deltas (when std is NaN due to single values) with 0 - observed_pentad["delta"] = observed_pentad["delta"].fillna(0.0) - observed_decad["delta"] = observed_decad["delta"].fillna(0.0) - - # Convert date columns to datetime - observed_pentad["date"] = pd.to_datetime(observed_pentad["date"]) - observed_decad["date"] = pd.to_datetime(observed_decad["date"]) - - # If there is a column name 'pentad', rename it to 'pentad_in_month' - if "pentad" in observed_pentad.columns: - observed_pentad.rename(columns={"pentad": "pentad_in_month"}, inplace=True) - logger.info( - f"Read {len(observed_pentad)} rows of observed data for the pentadal forecast horizon." - ) - - logger.info(f"Columns of observed pentad data: {observed_pentad.columns}") - logger.info(f"Columns of observed decad data: {observed_decad.columns}") - - # Read the predicted data - predicted_pentad = pd.read_csv(pred_pentad_path) - predicted_decad = pd.read_csv(pred_decad_path) - - # Convert date columns to datetime for predicted data - predicted_pentad["date"] = pd.to_datetime(predicted_pentad["date"]) - predicted_decad["date"] = pd.to_datetime(predicted_decad["date"]) - - logger.info(f"Columns of predicted pentad data: {predicted_pentad.columns}") - logger.info(f"Columns of predicted decad data: {predicted_decad.columns}") - - return observed_pentad, observed_decad, predicted_pentad, predicted_decad - - -def read_obs_pred_kgz(): - obs_pentad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/kyg_data_forecast_tools/intermediate_data/runoff_pentad.csv" - obs_decad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/kyg_data_forecast_tools/intermediate_data/runoff_decad.csv" - - pred_pentad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/kyg_data_forecast_tools/intermediate_data/combined_forecasts_pentad.csv" - pred_decad_path = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/data/kyg_data_forecast_tools/intermediate_data/combined_forecasts_decad.csv" - - # Read the observed data - observed_pentad = pd.read_csv(obs_pentad_path) - observed_decad = pd.read_csv(obs_decad_path) - - # Add a column model to the dataframe - observed_pentad["model_long"] = "Observed (Obs)" - observed_pentad["model_short"] = "Obs" - observed_decad["model_long"] = "Observed (Obs)" - observed_decad["model_short"] = "Obs" - - # Calculate delta (allowable uncertainty threshold) for each station and time period - # Delta = 0.674 * standard deviation of observed discharge for each code and pentad_in_year - observed_pentad["delta"] = observed_pentad.groupby(["code", "pentad_in_year"])[ - "discharge_avg" - ].transform(lambda x: 0.674 * x.std()) - observed_decad["delta"] = observed_decad.groupby(["code", "decad_in_year"])[ - "discharge_avg" - ].transform(lambda x: 0.674 * x.std()) - - # Fill NaN deltas (when std is NaN due to single values) with 0 - observed_pentad["delta"] = observed_pentad["delta"].fillna(0.0) - observed_decad["delta"] = observed_decad["delta"].fillna(0.0) - - # Convert date columns to datetime - observed_pentad["date"] = pd.to_datetime(observed_pentad["date"]) - observed_decad["date"] = pd.to_datetime(observed_decad["date"]) - - # If there is a column name 'pentad', rename it to 'pentad_in_month' - if "pentad" in observed_pentad.columns: - observed_pentad.rename(columns={"pentad": "pentad_in_month"}, inplace=True) - logger.info( - f"Read {len(observed_pentad)} rows of observed data for the pentadal forecast horizon." - ) - - logger.info(f"Columns of observed pentad data: {observed_pentad.columns}") - logger.info(f"Columns of observed decad data: {observed_decad.columns}") - # Read the predicted data - predicted_pentad = pd.read_csv(pred_pentad_path) - predicted_decad = pd.read_csv(pred_decad_path) - - # Convert date columns to datetime for predicted data - predicted_pentad["date"] = pd.to_datetime(predicted_pentad["date"]) - predicted_decad["date"] = pd.to_datetime(predicted_decad["date"]) - - logger.info(f"Columns of predicted pentad data: {predicted_pentad.columns}") - logger.info(f"Columns of predicted decad data: {predicted_decad.columns}") - - return observed_pentad, observed_decad, predicted_pentad, predicted_decad - - -def eval_tjk(eval_start=EVAL_START): - """ - Evaluate the skill metrics for the Tajikistan data. - """ - EVAL_START = eval_start - # Read the observed and predicted data - observed_pentad, observed_decad, predicted_pentad, predicted_decad = ( - read_obs_pred_tjk() - ) - - # Calculate the skill metrics for pentad - skill_stats_pentad, joint_forecasts_pentad, timing_stats = ( - calculate_skill_metrics_pentad(observed_pentad, predicted_pentad) - ) - - # Calculate the skill metrics for decad - skill_stats_decad, joint_forecasts_decad, timing_stats = ( - calculate_skill_metrics_decade(observed_decad, predicted_decad) - ) - - return skill_stats_pentad, skill_stats_decad - - -def eval_kgz(eval_start=EVAL_START): - """ - Evaluate the skill metrics for the Kyrgyzstan data. - """ - EVAL_START = eval_start - # Read the observed and predicted data - observed_pentad, observed_decad, predicted_pentad, predicted_decad = ( - read_obs_pred_kgz() - ) - - # Calculate the skill metrics for pentad - skill_stats_pentad, joint_forecasts_pentad, timing_stats = ( - calculate_skill_metrics_pentad(observed_pentad, predicted_pentad) - ) - - # Calculate the skill metrics for decad - skill_stats_decad, joint_forecasts_decad, timing_stats = ( - calculate_skill_metrics_decade(observed_decad, predicted_decad) - ) - - return skill_stats_pentad, skill_stats_decad - - -def main(): - # Evaluate the Tajikistan data - skill_stats_tjk = eval_tjk() - - # Evaluate the Kyrgyzstan data - skill_stats_kgz = eval_kgz() - - save_dir = "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/SAPPHIRE_Central_Asia_Technical_Work/code/machine_learning_hydrology/results_neural_discharge/sapp_skill_metrics/2023-2025" - - save_dir_tjk = os.path.join(save_dir, "tjk") - save_dir_kgz = os.path.join(save_dir, "kgz") - - if not os.path.exists(save_dir_tjk): - os.makedirs(save_dir_tjk) - - if not os.path.exists(save_dir_kgz): - os.makedirs(save_dir_kgz) - - # Save the skill stats for Tajikistan - skill_stats_tjk_pentad, skill_stats_tjk_decad = skill_stats_tjk - - skill_stats_tjk_pentad.to_csv( - os.path.join(save_dir_tjk, "skill_stats_pentad.csv"), index=False - ) - skill_stats_tjk_decad.to_csv( - os.path.join(save_dir_tjk, "skill_stats_decad.csv"), index=False - ) - - # Save the skill stats for Kyrgyzstan - skill_stats_kgz_pentad, skill_stats_kgz_decad = skill_stats_kgz - skill_stats_kgz_pentad.to_csv( - os.path.join(save_dir_kgz, "skill_stats_pentad.csv"), index=False - ) - skill_stats_kgz_decad.to_csv( - os.path.join(save_dir_kgz, "skill_stats_decad.csv"), index=False - ) - - -if __name__ == "__main__": - main() diff --git a/st_forecast/get_data.py b/st_forecast/get_data.py index 5c98f04..b6d08aa 100644 --- a/st_forecast/get_data.py +++ b/st_forecast/get_data.py @@ -856,7 +856,7 @@ def load_data_deprecated( df_rivers, df_era5, basins, success = load_data_df(path_config=data_config) if transform_mm: - print("Transform discharge from m3/s to mm/day") + logger.info("Transform discharge from m3/s to mm/day") for code in df_rivers["code"].unique(): mask = df_rivers["code"] == code if code not in basins.index: @@ -867,7 +867,7 @@ def load_data_deprecated( ) if transform_log: - print("Apply log1p transform to discharge") + logger.info("Apply log1p transform to discharge") df_rivers["discharge"] = np.log1p(df_rivers["discharge"]) if success["sla"]: @@ -880,7 +880,7 @@ def load_data_deprecated( df_era5 = df_era5.drop_duplicates(subset=["code", "date"], keep="last") - print("Columns ERA5:", df_era5.columns.tolist()) + logger.debug(f"Columns ERA5: {df_era5.columns.tolist()}") # filter out relevant basins codes_to_remove = data_config["codes_to_remove"] @@ -909,10 +909,10 @@ def load_data_deprecated( # ----------------- # if "train_years" in data_config and "test_years" in data_config: use_year_split = True - print("Using year-based splitting") + logger.info("Using year-based splitting") else: use_year_split = False - print("Using date-range based splitting") + logger.info("Using date-range based splitting") if use_year_split: train_years = data_config["train_years"] @@ -971,8 +971,7 @@ def load_data_deprecated( # this has the consequence that we can only predict on rivers that are in the training data codes_to_use = df_rivers_train["code"].unique() # Scale discharge (train) and apply to val/test using modular helpers - print("Scaling discharge data") - print("Using Global Scaling:", global_scaling) + logger.info(f"Scaling discharge data (global_scaling={global_scaling})") if scale_discharge_bool: df_rivers_train_scaled, scalers = compute_discharge_scalers( df_rivers_train, list(codes_to_use), "standard", global_scaling diff --git a/st_forecast/model_helper.py b/st_forecast/model_helper.py index 9dd2546..3f61c12 100644 --- a/st_forecast/model_helper.py +++ b/st_forecast/model_helper.py @@ -8,7 +8,6 @@ from pytorch_lightning.callbacks import Callback from pytorch_lightning.callbacks import EarlyStopping from pytorch_lightning.callbacks import StochasticWeightAveraging -import random from st_forecast.custom_models.FANMixer import FANMixerModel from st_forecast.custom_models.ExoLSTM import ExoLSTMModel @@ -42,12 +41,6 @@ def _resolve_encoder(encoder: str) -> dict | None: return _ENCODER_MAP[encoder] -# Set the random seed for reproducibility -random.seed(42) -torch.manual_seed(42) -np.random.seed(42) - - class LossLogger(Callback): def __init__(self): self.train_losses = [] diff --git a/st_forecast/operational/sapphire_predictor.py b/st_forecast/operational/sapphire_predictor.py index ac00b76..df24b92 100644 --- a/st_forecast/operational/sapphire_predictor.py +++ b/st_forecast/operational/sapphire_predictor.py @@ -4,7 +4,6 @@ """ import logging -import warnings from pathlib import Path import numpy as np @@ -29,16 +28,88 @@ load_model, ) -# Suppress verbose logging from pytorch_lightning -logging.getLogger("pytorch_lightning.utilities.rank_zero").setLevel(logging.WARNING) -logging.getLogger("pytorch_lightning.accelerators.cuda").setLevel(logging.WARNING) -logging.getLogger().setLevel(logging.WARNING) -logging.basicConfig(level=logging.WARNING) -warnings.filterwarnings("ignore") - logger = logging.getLogger(__name__) +def silence_pl_warnings() -> None: + """Opt-in helper: tames the noisiest pytorch_lightning loggers. + + Call from your application entrypoint if you want quiet predictions. + Importing :class:`SapphirePredictor` no longer mutates global logging + state — that broke downstream apps that configured their own logging. + """ + logging.getLogger("pytorch_lightning.utilities.rank_zero").setLevel( + logging.WARNING + ) + logging.getLogger("pytorch_lightning.accelerators.cuda").setLevel(logging.WARNING) + + +class SapphirePredictionError(RuntimeError): + """Raised when SapphirePredictor.predict / .hindcast fails. + + Wraps the underlying exception with basin-and-horizon context so callers + can catch a single class and report meaningfully. + """ + + +def _coerce_basin_code(identifier: int | str) -> int: + """Coerce a basin identifier to int with a clear error on bad input.""" + if isinstance(identifier, int): + return identifier + try: + return int(str(identifier).strip()) + except (ValueError, TypeError) as e: + raise ValueError( + f"Basin identifier {identifier!r} must be an integer or a numeric " + f"string; got {type(identifier).__name__}." + ) from e + + +def _validate_predict_inputs( + past_measurements: pd.DataFrame, + covariates: pd.DataFrame, + code: int, + config, +) -> None: + """Fail fast with a clear message instead of crashing deep inside Darts.""" + date_col = config.date_col + target_col = config.target_col + basin_id_col = config.basin_id_col + + for name, df in (("past_measurements", past_measurements), ("covariates", covariates)): + if not isinstance(df, pd.DataFrame): + raise TypeError( + f"{name} must be a pandas DataFrame, got {type(df).__name__}." + ) + if df.empty: + raise ValueError(f"{name} is empty for basin {code}.") + if date_col not in df.columns: + raise ValueError( + f"{name} is missing required date column {date_col!r} " + f"(available: {list(df.columns)})." + ) + + if target_col not in past_measurements.columns: + raise ValueError( + f"past_measurements is missing target column {target_col!r} " + f"(available: {list(past_measurements.columns)})." + ) + if past_measurements[target_col].isna().any(): + n_nan = int(past_measurements[target_col].isna().sum()) + raise ValueError( + f"past_measurements[{target_col!r}] contains {n_nan} NaN value(s) " + f"for basin {code}; predictor requires a clean target series." + ) + + if basin_id_col in past_measurements.columns: + unique_codes = past_measurements[basin_id_col].unique() + if len(unique_codes) > 1: + raise ValueError( + f"past_measurements contains multiple basins {list(unique_codes)}; " + f"predictor expects pre-filtered single-basin data." + ) + + class SapphirePredictor(BasePredictor): """Darts-based Deep Learning predictor for SAPPHIRE discharge forecasting. @@ -273,13 +344,16 @@ def predict( Predictions with columns: date, code, Q5, Q10, Q25, Q50, Q75, Q90, Q95 """ if n > self.config.forecast_horizon: - logging.warning( + logger.warning( f"Requested horizon n={n} exceeds output_chunk_length=" f"{self.config.forecast_horizon}. Capping to " f"{self.config.forecast_horizon}." ) n = self.config.forecast_horizon - code = int(identifier) + code = _coerce_basin_code(identifier) + _validate_predict_inputs( + past_measurements, covariates, code, self.config + ) # Prepare DataFrames df_discharge, df_covariates, df_covariates_past = self._prepare_data( @@ -293,15 +367,20 @@ def predict( # Run model.predict() trainer = Trainer(accelerator="cpu", logger=False) - prediction = self.model.predict( - n=n, - series=target_ts, - past_covariates=past_cov_ts, - future_covariates=future_cov_ts, - num_samples=self.num_samples, - trainer=trainer, - mc_dropout=self.config.mc_dropout, - ) + try: + prediction = self.model.predict( + n=n, + series=target_ts, + past_covariates=past_cov_ts, + future_covariates=future_cov_ts, + num_samples=self.num_samples, + trainer=trainer, + mc_dropout=self.config.mc_dropout, + ) + except Exception as e: + raise SapphirePredictionError( + f"model.predict failed for basin {code} (n={n}): {e}" + ) from e # Post-process: quantiles + inverse scaling pred_df = create_prediction_df( @@ -341,13 +420,16 @@ def hindcast( date, Q5..Q95, forecast_date """ if n > self.config.forecast_horizon: - logging.warning( + logger.warning( f"Requested horizon n={n} exceeds output_chunk_length=" f"{self.config.forecast_horizon}. Capping to " f"{self.config.forecast_horizon}." ) n = self.config.forecast_horizon - code = int(identifier) + code = _coerce_basin_code(identifier) + _validate_predict_inputs( + past_measurements, covariates, code, self.config + ) date_col = self.config.date_col # Prepare DataFrames (same as predict — no mode distinction) @@ -365,18 +447,23 @@ def hindcast( "trainer": Trainer(accelerator="cpu", logger=False), "mc_dropout": self.config.mc_dropout, } - backtests = self.model.historical_forecasts( - series=target_ts, - past_covariates=past_cov_ts, - future_covariates=future_cov_ts, - forecast_horizon=n, - num_samples=self.num_samples, - retrain=False, - stride=1, - last_points_only=False, - verbose=False, - predict_kwargs=predict_kwargs, - ) + try: + backtests = self.model.historical_forecasts( + series=target_ts, + past_covariates=past_cov_ts, + future_covariates=future_cov_ts, + forecast_horizon=n, + num_samples=self.num_samples, + retrain=False, + stride=1, + last_points_only=False, + verbose=False, + predict_kwargs=predict_kwargs, + ) + except Exception as e: + raise SapphirePredictionError( + f"model.historical_forecasts failed for basin {code} (n={n}): {e}" + ) from e # Process results — match BaseDartsDLPredictor output schema: # date, Q columns, forecast_date diff --git a/st_forecast/pipeline/forecast.py b/st_forecast/pipeline/forecast.py index b03d6a2..4b4a76f 100644 --- a/st_forecast/pipeline/forecast.py +++ b/st_forecast/pipeline/forecast.py @@ -226,6 +226,7 @@ def _run_predictions( logger.info(f"Generating {n}-day forecasts for {len(target_series)} basins...") all_predictions = [] + failed_basins: list[tuple[int | str, str]] = [] for i, (target, code) in enumerate(zip(target_series, basin_codes)): past_cov = past_covariates[i] if past_covariates else None @@ -247,13 +248,24 @@ def _run_predictions( all_predictions.append(pred_df) except Exception as e: - logger.warning(f"Failed to generate forecast for basin {code}: {e}") + logger.exception(f"Failed to generate forecast for basin {code}") + failed_basins.append((code, repr(e))) continue if not all_predictions: - raise ValueError("No successful predictions generated") + raise ValueError( + f"No successful predictions generated; " + f"all {len(failed_basins)} basin(s) failed." + ) result = pd.concat(all_predictions, ignore_index=True) + if failed_basins: + logger.warning( + f"Forecast completed with {len(failed_basins)} failed basin(s) " + f"out of {len(target_series)}: {[c for c, _ in failed_basins]}" + ) + # Stash failures on the DataFrame so callers can inspect partial-success runs. + result.attrs["failed_basins"] = failed_basins logger.info(f"Generated {len(result)} prediction rows") return result diff --git a/st_forecast/pipeline/inference.py b/st_forecast/pipeline/inference.py index c02ac1c..23dad59 100644 --- a/st_forecast/pipeline/inference.py +++ b/st_forecast/pipeline/inference.py @@ -380,11 +380,15 @@ def inverse_scale_predictions( # Step 1: Inverse standard scaling (if applicable) if scalers.discharge is not None: for code in result_df[basin_col].unique(): - if code not in scalers.discharge: - logger.warning(f"No scaler found for code {code}, skipping unscaling") - continue + scaler_entry = _lookup_basin_scaler(scalers.discharge, code) + if scaler_entry is None: + raise KeyError( + f"No discharge scaler found for basin {code!r}; available " + f"scaler keys (sample): {list(scalers.discharge)[:10]}. " + f"Returning predictions unscaled would yield wrong magnitudes." + ) - mean, std = scalers.discharge[code] + mean, std = scaler_entry mask = result_df[basin_col] == code for col in q_cols: @@ -398,13 +402,11 @@ def inverse_scale_predictions( # Step 3: Convert mm to m3/s if config.transform_mm: for code in result_df[basin_col].unique(): - if code not in static_df.index: - logger.warning( - f"Code {code} not found in static_df for mm transformation" + area_km2 = _lookup_static_field(static_df, code, "area_km2") + if area_km2 is None: + raise KeyError( + f"Basin {code!r} not found in static_df for mm→m3/s conversion." ) - continue - - area_km2 = static_df.loc[code, "area_km2"] mask = result_df[basin_col] == code for col in q_cols: @@ -413,6 +415,46 @@ def inverse_scale_predictions( return result_df +def _lookup_basin_scaler(scalers_dict: dict, code) -> tuple | None: + """Try common basin-code key variants (int, np.int64-cast, str) before giving up. + + Scaler keys are persisted via ``_load_scaler_csv``: digit-like indices become + int, others stay str. User-supplied DataFrames sometimes produce ``np.int64`` + or string codes — without normalization, the lookup silently misses and + predictions are returned in scaled units. + """ + if code in scalers_dict: + return scalers_dict[code] + try: + as_int = int(code) + if as_int in scalers_dict: + return scalers_dict[as_int] + except (ValueError, TypeError): + pass + as_str = str(code) + if as_str in scalers_dict: + return scalers_dict[as_str] + return None + + +def _lookup_static_field(static_df, code, field: str): + """Same robustness for static_df.loc[code, field] lookups.""" + for candidate in _candidate_keys(code): + if candidate in static_df.index: + return static_df.loc[candidate, field] + return None + + +def _candidate_keys(code): + """Yield basin-code key variants in priority order.""" + yield code + try: + yield int(code) + except (ValueError, TypeError): + pass + yield str(code) + + def save_predictions( predictions_df: pd.DataFrame, output_folder: Path, diff --git a/st_forecast/pipeline/workflow.py b/st_forecast/pipeline/workflow.py index 247fb54..c8a8b44 100644 --- a/st_forecast/pipeline/workflow.py +++ b/st_forecast/pipeline/workflow.py @@ -13,6 +13,7 @@ ) from st_forecast.pipeline.artifacts import RunConfig, Scalers from st_forecast.pipeline.training import TrainingResult, train +from st_forecast.training_components import set_seed logger = logging.getLogger(__name__) @@ -35,6 +36,7 @@ def train_model( val_years: list[int] | None = None, train_filter: pd.Series | None = None, val_filter: pd.Series | None = None, + seed: int | None = 42, ) -> TrainModelResult: """Train a discharge forecasting model with a single function call. @@ -112,6 +114,10 @@ def train_model( output_path = Path(output_path) output_path.mkdir(parents=True, exist_ok=True) + if seed is not None: + set_seed(seed) + logger.info(f"Random seed set to {seed}") + # Ensure date column is datetime date_col = config.date_col if not pd.api.types.is_datetime64_any_dtype(temporal_df[date_col]): diff --git a/st_forecast/training_components.py b/st_forecast/training_components.py index 92dfcc8..b712636 100644 --- a/st_forecast/training_components.py +++ b/st_forecast/training_components.py @@ -2,8 +2,12 @@ from __future__ import annotations +import logging +import random from typing import TYPE_CHECKING +import numpy as np +import torch import torch.optim.lr_scheduler as lr_scheduler from darts.utils.likelihood_models import ( CauchyLikelihood, @@ -18,6 +22,27 @@ if TYPE_CHECKING: from st_forecast.pipeline.artifacts import RunConfig +logger = logging.getLogger(__name__) + + +def set_seed(seed: int) -> None: + """Seed Python, NumPy, PyTorch (CPU + CUDA), and Lightning workers. + + Call once from the entrypoint (CLI or programmatic) — *not* at module + import time. Importing this package no longer reseeds the user's RNG. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + try: + import lightning.pytorch as pl + + pl.seed_everything(seed, workers=True) + except ImportError: + logger.debug("lightning.pytorch unavailable; skipped pl.seed_everything") + class LossLogger(Callback): def __init__(self): diff --git a/tests/test_inference.py b/tests/test_inference.py index 443d72b..5f4bb5c 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -423,27 +423,32 @@ def test_combined_transforms( expected = (expected / 1000) * (1000.0 * 1e6) / 86400 # After step 3 assert result["Q50"].iloc[0] == pytest.approx(expected, rel=1e-5) - def test_missing_scaler_warning(self, sample_predictions_df, sample_config, caplog): - """Test that missing scaler for a code logs a warning.""" - # Create scalers without code 1002 + def test_missing_scaler_raises(self, sample_predictions_df, sample_config): + """Missing scaler must fail loudly — silently skipping yields wrong magnitudes.""" incomplete_scalers = Scalers( discharge={1001: (10.0, 5.0)}, discharge_scaler_type="standard" ) - with caplog.at_level("WARNING"): - result = inverse_scale_predictions( + with pytest.raises(KeyError, match="No discharge scaler found for basin"): + inverse_scale_predictions( sample_predictions_df, incomplete_scalers, sample_config ) - # Should log warning about missing code 1002 - assert any( - "No scaler found for code 1002" in record.message - for record in caplog.records + def test_scaler_lookup_handles_int_str_drift(self, sample_config): + """Predictions with str basin codes still find int-keyed scalers.""" + df = pd.DataFrame( + { + "date": pd.date_range("2024-01-01", periods=2), + "code": ["1001", "1001"], + "Q50": [1.0, 2.0], + "forecast_step": [1, 2], + } ) - - # Code 1001 should be transformed, code 1002 should remain unchanged - assert result[result["code"] == 1001]["Q50"].iloc[0] != 1.0 # Transformed - assert result[result["code"] == 1002]["Q50"].iloc[0] == 1.5 # Unchanged + scalers = Scalers( + discharge={1001: (10.0, 5.0)}, discharge_scaler_type="standard" + ) + result = inverse_scale_predictions(df, scalers, sample_config) + assert result["Q50"].iloc[0] == pytest.approx(1.0 * 5.0 + 10.0) def test_missing_static_df_for_mm_transform( self, sample_predictions_df, sample_scalers, sample_config @@ -456,31 +461,25 @@ def test_missing_static_df_for_mm_transform( sample_predictions_df, sample_scalers, sample_config, static_df=None ) - def test_missing_code_in_static_df( - self, sample_predictions_df, sample_scalers, sample_config, caplog + def test_missing_code_in_static_df_raises( + self, sample_predictions_df, sample_scalers, sample_config ): - """Test that missing code in static_df for mm transform logs warning.""" + """Missing basin in static_df during mm transform must fail loudly.""" sample_config.transform_mm = True - # Create static_df without code 1002 incomplete_static = pd.DataFrame( {"CODE": [1001], "area_km2": [1000.0], "elevation": [1500.0]} ) incomplete_static.index = incomplete_static["CODE"] - with caplog.at_level("WARNING"): - result = inverse_scale_predictions( + with pytest.raises(KeyError, match="not found in static_df"): + inverse_scale_predictions( sample_predictions_df, sample_scalers, sample_config, incomplete_static, ) - assert any( - "Code 1002 not found in static_df" in record.message - for record in caplog.records - ) - def test_no_scaling_when_scaler_is_none(self, sample_predictions_df, sample_config): """Test that predictions are unchanged when no scalers are provided.""" no_scalers = Scalers(discharge=None) diff --git a/tests/test_public_api_smoke.py b/tests/test_public_api_smoke.py new file mode 100644 index 0000000..bfc76c9 --- /dev/null +++ b/tests/test_public_api_smoke.py @@ -0,0 +1,431 @@ +"""Regression tests for the publication-readiness fixes. + +These check that the public surface stays well-behaved at import time and +that the `SapphirePredictor` input-validation helpers reject bad input +loudly instead of failing deep inside Darts. +""" + +import logging +import random +import tempfile +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + + +class TestPublicApiSurface: + def test_every_documented_export_is_importable(self): + """README advertises these names — they must import from package root.""" + from st_forecast import ( + BASIN_ID_COL, + ColumnConfig, + DATE_COL, + DEFAULT_QUANTILES, + RunConfig, + SapphirePredictionError, + SapphirePredictor, + TARGET_COL, + TrainModelResult, + __version__, + silence_pl_warnings, + train_model, + ) + + assert isinstance(__version__, str) + assert isinstance(DEFAULT_QUANTILES, list) and len(DEFAULT_QUANTILES) > 0 + assert callable(silence_pl_warnings) + assert callable(train_model) + assert issubclass(SapphirePredictionError, RuntimeError) + assert BASIN_ID_COL == "code" + assert DATE_COL == "date" + assert TARGET_COL == "discharge" + + +class TestImportSideEffects: + def test_importing_predictor_does_not_mutate_root_logger(self): + """Library code must never call logging.basicConfig or set the root level.""" + logging.basicConfig(level=logging.DEBUG) + before = logging.getLogger().level + + # Force re-import to test the import path even after earlier tests. + import importlib + + import st_forecast.operational.sapphire_predictor as sp + + importlib.reload(sp) + + after = logging.getLogger().level + assert before == after, ( + f"Importing SapphirePredictor mutated root logger: {before} -> {after}" + ) + + def test_importing_model_helper_does_not_reseed_user_rng(self): + """model_helper used to call random.seed(42) at import time, which + clobbered any seed the user had set programmatically before importing. + """ + random.seed(0) + expected = random.random() + + import importlib + + import st_forecast.model_helper as mh + + importlib.reload(mh) + + random.seed(0) + got = random.random() + assert got == expected, ( + "Importing st_forecast.model_helper mutated user RNG state." + ) + + +class TestSetSeed: + def test_set_seed_is_deterministic_across_calls(self): + from st_forecast.training_components import set_seed + + set_seed(42) + a = random.random() + set_seed(42) + b = random.random() + assert a == b + + +class TestBasinCodeCoercion: + def test_int_pass_through(self): + from st_forecast.operational.sapphire_predictor import _coerce_basin_code + + assert _coerce_basin_code(1234) == 1234 + + def test_numeric_string_coerces_to_int(self): + from st_forecast.operational.sapphire_predictor import _coerce_basin_code + + assert _coerce_basin_code("1234") == 1234 + assert _coerce_basin_code(" 1234 ") == 1234 + + def test_non_numeric_string_raises_with_clear_message(self): + from st_forecast.operational.sapphire_predictor import _coerce_basin_code + + with pytest.raises(ValueError, match="must be an integer or a numeric string"): + _coerce_basin_code("GAUGE-001") + + +class TestValidatePredictInputs: + """`_validate_predict_inputs` is the gatekeeper for predict() / hindcast(). + + These tests pin its behavior so future refactors can't silently regress + into deep-Darts crashes. + """ + + def _make_config(self): + from types import SimpleNamespace + + return SimpleNamespace( + date_col="date", + target_col="discharge", + basin_id_col="code", + ) + + def _good_df(self): + return pd.DataFrame( + { + "date": pd.date_range("2024-01-01", periods=5), + "code": [1] * 5, + "discharge": [1.0, 2.0, 3.0, 2.5, 2.0], + } + ) + + def test_accepts_clean_input(self): + from st_forecast.operational.sapphire_predictor import _validate_predict_inputs + + df = self._good_df() + _validate_predict_inputs(df, df, code=1, config=self._make_config()) + + def test_rejects_non_dataframe(self): + from st_forecast.operational.sapphire_predictor import _validate_predict_inputs + + with pytest.raises(TypeError, match="must be a pandas DataFrame"): + _validate_predict_inputs( + [1, 2, 3], + self._good_df(), + code=1, + config=self._make_config(), + ) + + def test_rejects_empty_dataframe(self): + from st_forecast.operational.sapphire_predictor import _validate_predict_inputs + + with pytest.raises(ValueError, match="is empty for basin 1"): + _validate_predict_inputs( + pd.DataFrame(columns=["date", "code", "discharge"]), + self._good_df(), + code=1, + config=self._make_config(), + ) + + def test_rejects_missing_date_column(self): + from st_forecast.operational.sapphire_predictor import _validate_predict_inputs + + bad = self._good_df().rename(columns={"date": "timestamp"}) + with pytest.raises(ValueError, match="missing required date column"): + _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + + def test_rejects_missing_target_column(self): + from st_forecast.operational.sapphire_predictor import _validate_predict_inputs + + bad = self._good_df().drop(columns=["discharge"]) + with pytest.raises(ValueError, match="missing target column"): + _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + + def test_rejects_nan_in_target(self): + from st_forecast.operational.sapphire_predictor import _validate_predict_inputs + + bad = self._good_df() + bad.loc[2, "discharge"] = float("nan") + with pytest.raises(ValueError, match="contains 1 NaN value"): + _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + + def test_rejects_multi_basin_input(self): + from st_forecast.operational.sapphire_predictor import _validate_predict_inputs + + bad = self._good_df() + bad.loc[0, "code"] = 999 + with pytest.raises(ValueError, match="multiple basins"): + _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + + +def _synthetic_basin_data( + seed: int = 0, + basin_codes: tuple[int, int] = (1001, 1002), + start: str = "2020-01-01", + n_days: int = 730, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Generate a tiny self-contained 2-basin dataset for the smoke test. + + Discharge has a clear seasonal signal so a 1-epoch model still produces + finite, sane-magnitude predictions. Static features include the columns + the predictor's feature engineering may consult (LAT, LON, area_km2). + """ + rng = np.random.default_rng(seed) + dates = pd.date_range(start, periods=n_days, freq="D") + rows = [] + for code in basin_codes: + day_of_year = dates.dayofyear.values + season = np.sin(2 * np.pi * day_of_year / 365.25) + # Per-basin scale so predictions are in different magnitude bands — + # this is the regression for H1 (scaler key drift). + scale = 5.0 if code == basin_codes[0] else 50.0 + discharge = scale * (1.5 + season + 0.3 * rng.standard_normal(n_days)) + discharge = np.clip(discharge, 0.01, None) # strictly positive + temperature = 10.0 + 15.0 * season + rng.standard_normal(n_days) + precipitation = np.clip( + 2.0 + 3.0 * (season + 1) + rng.standard_normal(n_days), 0.0, None + ) + rows.append( + pd.DataFrame( + { + "date": dates, + "code": code, + "discharge": discharge, + "T": temperature, + "P": precipitation, + } + ) + ) + temporal_df = pd.concat(rows, ignore_index=True) + + static_df = pd.DataFrame( + { + "CODE": list(basin_codes), + "area_km2": [200.0, 1500.0], + "LAT": [42.5, 41.0], + "LON": [74.0, 73.0], + "mean_elev_m": [1500.0, 2500.0], + } + ) + static_df.index = static_df["CODE"] + return temporal_df, static_df + + +@pytest.mark.slow +class TestEndToEndTrainPredict: + """Real train→save→reload→predict smoke test on synthetic data. + + Marked ``slow`` so users can opt out via ``pytest -m 'not slow'``. On CPU + it should finish in ~15–30 s. The assertions focus on correctness + properties that the publication blockers were about, not on accuracy: + no module-import side effects, predictions in physical (unscaled) units, + failed-basin attrs surface, and so on. + """ + + def _build_config(self, work_dir: Path): + from st_forecast import RunConfig + + return RunConfig( + region="synthetic", + model_type="TiDE", + model_name="tide_smoke", + input_chunk_length=30, + forecast_horizon=5, + exog_features=["T", "P"], + past_features=[], + future_features=["T", "P"], + static_features=["area_km2", "mean_elev_m"], + n_epochs=1, + hidden_size=8, + temporal_decoder_hidden_size=8, + decoder_output_dim=4, + dropout=0.0, + mc_dropout=False, + batch_size=64, + lr=1e-3, + num_samples=20, + scale_discharge_bool=True, + transform_log=False, + transform_mm=False, + early_stopping_enabled=False, + use_only_natural_rivers=False, + accelerator="cpu", + work_dir=str(work_dir), + ) + + def test_train_save_reload_predict(self): + from st_forecast import SapphirePredictor, train_model + + temporal_df, static_df = _synthetic_basin_data() + + with tempfile.TemporaryDirectory(prefix="st_forecast_smoke_") as tmp: + work_dir = Path(tmp) + config = self._build_config(work_dir) + + result = train_model( + temporal_df=temporal_df, + static_df=static_df, + config=config, + output_path=work_dir, + train_years=[2020], + val_years=[2021], + seed=0, + ) + + # Artifact bundle invariants — what SapphirePredictor relies on. + assert (work_dir / "config.json").exists() + assert (work_dir / "static_df.csv").exists() + assert (work_dir / "scalers" / "discharge_scalers.csv").exists() + assert (work_dir / f"{config.model_name}.pt").exists() + assert result.scalers.discharge is not None + assert set(result.scalers.discharge.keys()) == {1001, 1002} + + # Reload through the documented public surface. + predictor = SapphirePredictor(str(work_dir)) + assert predictor.config.model_type == "TiDE" + + # Build a single-basin slice covering input_chunk_length history, + # plus future covariates extending forecast_horizon days past it. + target_code = 1002 # the larger-magnitude basin + history_end = pd.Timestamp("2021-06-01") + past_window = ( + temporal_df["date"] <= history_end + ) & (temporal_df["code"] == target_code) + future_window = ( + (temporal_df["date"] > history_end) + & ( + temporal_df["date"] + <= history_end + pd.Timedelta(days=config.forecast_horizon) + ) + & (temporal_df["code"] == target_code) + ) + past_measurements = temporal_df.loc[ + past_window, ["date", "code", "discharge"] + ].copy() + covariates = temporal_df.loc[ + past_window | future_window, ["date", "code", "T", "P"] + ].copy() + + preds = predictor.predict( + past_measurements=past_measurements, + covariates=covariates, + identifier=target_code, + n=config.forecast_horizon, + ) + + # Output schema. + assert isinstance(preds, pd.DataFrame) + assert "date" in preds.columns + for q in ("Q5", "Q50", "Q95"): + assert q in preds.columns, f"missing quantile column {q}" + assert len(preds) == config.forecast_horizon + + # Predictions must be finite (regression for B5/H4: silenced + # warnings used to mask NaN-producing failures). + for q in ("Q5", "Q50", "Q95"): + assert preds[q].notna().all(), f"{q} contains NaN" + assert np.isfinite(preds[q]).all(), f"{q} contains inf" + + # H1 regression: predictions must come back in *physical* units, + # not scaled units. The training data for basin 1002 had a mean + # near 50 * 1.5 = 75 with seasonal swings to 0–150-ish. Scaled + # values would sit near 0 with stdev ~1, so a sane lower bound + # rules out the silent-skip-unscaling bug. + assert preds["Q50"].abs().mean() > 5.0, ( + f"Q50 magnitudes look like scaled units, not physical units: " + f"{preds['Q50'].tolist()}" + ) + # Quantile ordering: Q5 ≤ Q50 ≤ Q95 row-wise. + assert (preds["Q5"] <= preds["Q50"] + 1e-6).all() + assert (preds["Q50"] <= preds["Q95"] + 1e-6).all() + + def test_predict_with_string_basin_id_still_unscales(self): + """H1 + H2 regression: passing the basin code as a numeric string must + still locate the int-keyed scaler and return physical units.""" + from st_forecast import SapphirePredictor, train_model + + temporal_df, static_df = _synthetic_basin_data() + + with tempfile.TemporaryDirectory(prefix="st_forecast_smoke_") as tmp: + work_dir = Path(tmp) + config = self._build_config(work_dir) + + train_model( + temporal_df=temporal_df, + static_df=static_df, + config=config, + output_path=work_dir, + train_years=[2020], + val_years=[2021], + seed=0, + ) + + predictor = SapphirePredictor(str(work_dir)) + + history_end = pd.Timestamp("2021-06-01") + past_window = ( + temporal_df["date"] <= history_end + ) & (temporal_df["code"] == 1002) + future_window = ( + (temporal_df["date"] > history_end) + & ( + temporal_df["date"] + <= history_end + pd.Timedelta(days=config.forecast_horizon) + ) + & (temporal_df["code"] == 1002) + ) + past_measurements = temporal_df.loc[ + past_window, ["date", "code", "discharge"] + ].copy() + covariates = temporal_df.loc[ + past_window | future_window, ["date", "code", "T", "P"] + ].copy() + + preds = predictor.predict( + past_measurements=past_measurements, + covariates=covariates, + identifier="1002", # <-- string, not int + n=config.forecast_horizon, + ) + + assert preds["Q50"].abs().mean() > 5.0, ( + "String basin ID lookup did not find int-keyed scaler — " + "predictions are still in scaled units." + ) From 0ebc410fdf051b7635539c5089ffde8b8c143e33 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Wed, 27 May 2026 15:16:44 +0200 Subject: [PATCH 09/20] sapphire pred --- st_forecast/cli/train.py | 12 +- st_forecast/operational/sapphire_predictor.py | 55 +++-- tests/test_public_api_smoke.py | 194 +++++++++++++++++- 3 files changed, 236 insertions(+), 25 deletions(-) diff --git a/st_forecast/cli/train.py b/st_forecast/cli/train.py index 4af6ff4..15bf600 100644 --- a/st_forecast/cli/train.py +++ b/st_forecast/cli/train.py @@ -1,11 +1,13 @@ """CLI command for training the forecasting model.""" import logging +import random import shutil import click import numpy as np import pandas as pd +import torch from pathlib import Path from st_forecast import get_data @@ -17,11 +19,17 @@ ) from st_forecast.pipeline.artifacts import Scalers, load_run_artifacts from st_forecast.pipeline.training import train -from st_forecast.training_components import set_seed logger = logging.getLogger(__name__) +def _set_seeds(seed: int) -> None: + """Set random seeds for reproducibility.""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + def _save_losses( train_losses: list[float], val_losses: list[float], @@ -69,7 +77,7 @@ def train_cli(run_folder: Path, config: Path | None, seed: int) -> None: shutil.copy(config, config_dest) logger.info(f"Copied config from {config} to {config_dest}") - set_seed(seed) + _set_seeds(seed) logger.info(f"Random seed set to {seed}") # Load artifacts diff --git a/st_forecast/operational/sapphire_predictor.py b/st_forecast/operational/sapphire_predictor.py index df24b92..e1aa1dd 100644 --- a/st_forecast/operational/sapphire_predictor.py +++ b/st_forecast/operational/sapphire_predictor.py @@ -96,9 +96,11 @@ def _validate_predict_inputs( ) if past_measurements[target_col].isna().any(): n_nan = int(past_measurements[target_col].isna().sum()) - raise ValueError( + logger.warning( f"past_measurements[{target_col!r}] contains {n_nan} NaN value(s) " - f"for basin {code}; predictor requires a clean target series." + f"for basin {code}; hindcasts will be skipped for windows that " + f"overlap a NaN, and predict() will return NaN if the tail is " + f"contaminated." ) if basin_id_col in past_measurements.columns: @@ -215,7 +217,7 @@ def _prepare_data( basin_id_col=basin_id_col, ) - # Unit-transform discharge (mm conversion, log transform) + # Unit-transform discharge (mm conversion, log transform). if self.config.transform_mm or self.config.transform_log: df_discharge = transform_discharge_units( df_discharge, @@ -224,7 +226,23 @@ def _prepare_data( self.config.transform_log, ) - # Scale discharge + # Compute past covariates (rolling means) on the post-transform but + # PRE-SCALING discharge. This order must match the training pipeline + # in `_build_timeseries_pipeline` (moving averages added before + # `compute_discharge_scalers`); otherwise inference feeds the model + # past-covariates in a different magnitude band than it was trained + # on, biasing predictions and dropping NSE. + df_covariates_past = pd.DataFrame({date_col: df_discharge[date_col].values}) + for window in self.config.moving_windows: + rolling = df_discharge[target_col].rolling(window=window).mean() + if self.config.shift_moving_avg: + rolling = rolling.shift(periods=self.config.forecast_horizon) + rolling = rolling.bfill() + suffix = "_shifted" if self.config.shift_moving_avg else "" + col = f"moving_avr_dis_{window}{suffix}" + df_covariates_past[col] = rolling.values + + # Scale discharge (after moving-average computation). if self.config.scale_discharge_bool and self.scalers.discharge: df_discharge = apply_discharge_scalers( df_discharge, @@ -233,7 +251,7 @@ def _prepare_data( self.config.global_scaling, ) - # Scale covariates (exog features) + # Scale covariates (exog features). if self.config.exog_features and self.scalers.era5: df_covariates = apply_era5_scalers( df_covariates, @@ -242,17 +260,6 @@ def _prepare_data( self.config.forcing_scaler_type, ) - # Compute past covariates (rolling means on scaled discharge) - df_covariates_past = pd.DataFrame({date_col: df_discharge[date_col].values}) - for window in self.config.moving_windows: - rolling = df_discharge[target_col].rolling(window=window).mean() - if self.config.shift_moving_avg: - rolling = rolling.shift(periods=self.config.forecast_horizon) - rolling = rolling.bfill() - suffix = "_shifted" if self.config.shift_moving_avg else "" - col = f"moving_avr_dis_{window}{suffix}" - df_covariates_past[col] = rolling.values - return df_discharge, df_covariates, df_covariates_past def _create_time_series( @@ -417,7 +424,13 @@ def hindcast( ------- pd.DataFrame Hindcast predictions with columns: - date, Q5..Q95, forecast_date + ``date, code, forecast_issue_date, forecast_step, Q5..Q95``. + ``forecast_issue_date`` is the day before the first prediction + (the forecast origin); ``forecast_step`` is the days-ahead index + (1..n). Each target ``date`` appears in up to ``n`` overlapping + windows, one per step — disambiguate with ``forecast_step`` when + joining against observations. Schema matches + :func:`st_forecast.pipeline.hindcast.hindcast`. """ if n > self.config.forecast_horizon: logger.warning( @@ -465,13 +478,13 @@ def hindcast( f"model.historical_forecasts failed for basin {code} (n={n}): {e}" ) from e - # Process results — match BaseDartsDLPredictor output schema: - # date, Q columns, forecast_date + # Process results — schema matches pipeline.hindcast.hindcast(): + # date, code, forecast_issue_date, forecast_step, Q5..Q95 all_dfs = [] for ts in backtests: df = create_prediction_df(ts, self.quantiles, date_col=date_col) - min_date = df[date_col].min() - df["forecast_date"] = min_date - pd.DateOffset(days=1) + df["forecast_issue_date"] = df[date_col].min() - pd.DateOffset(days=1) + df["forecast_step"] = np.arange(1, len(df) + 1) all_dfs.append(df) result = pd.concat(all_dfs, ignore_index=True) diff --git a/tests/test_public_api_smoke.py b/tests/test_public_api_smoke.py index bfc76c9..8b7b98d 100644 --- a/tests/test_public_api_smoke.py +++ b/tests/test_public_api_smoke.py @@ -178,13 +178,19 @@ def test_rejects_missing_target_column(self): with pytest.raises(ValueError, match="missing target column"): _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) - def test_rejects_nan_in_target(self): + def test_warns_on_nan_in_target(self, caplog): + """NaN in target is a warning, not an error: Darts skips hindcast + windows that touch NaN, and predict() returns NaN if the tail is + contaminated. Both are valid downstream behaviors.""" from st_forecast.operational.sapphire_predictor import _validate_predict_inputs bad = self._good_df() bad.loc[2, "discharge"] = float("nan") - with pytest.raises(ValueError, match="contains 1 NaN value"): + with caplog.at_level("WARNING", logger="st_forecast.operational.sapphire_predictor"): _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + assert any( + "contains 1 NaN value" in record.message for record in caplog.records + ), f"expected NaN warning, got: {[r.message for r in caplog.records]}" def test_rejects_multi_basin_input(self): from st_forecast.operational.sapphire_predictor import _validate_predict_inputs @@ -429,3 +435,187 @@ def test_predict_with_string_basin_id_still_unscales(self): "String basin ID lookup did not find int-keyed scaler — " "predictions are still in scaled units." ) + + def _build_config_with_moving_avgs(self, work_dir: Path): + """Variant of _build_config that exercises the past_features / + moving_avr_dis_N code path. The training-vs-inference + moving-average bug only manifests when this is non-empty. + """ + from st_forecast import RunConfig + + return RunConfig( + region="synthetic", + model_type="TiDE", + model_name="tide_smoke_mavg", + input_chunk_length=30, + forecast_horizon=5, + exog_features=["T", "P"], + past_features=["moving_avr_dis_3", "moving_avr_dis_5"], + future_features=["T", "P"], + static_features=["area_km2", "mean_elev_m"], + moving_windows=[3, 5], + shift_moving_avg=False, + n_epochs=1, + hidden_size=8, + temporal_decoder_hidden_size=8, + decoder_output_dim=4, + dropout=0.0, + mc_dropout=False, + batch_size=64, + lr=1e-3, + num_samples=20, + scale_discharge_bool=True, + transform_log=False, + transform_mm=False, + early_stopping_enabled=False, + use_only_natural_rivers=False, + accelerator="cpu", + work_dir=str(work_dir), + ) + + def test_predictor_and_pipeline_hindcast_agree_on_q50(self): + """Regression for the train/inference moving-average ordering bug. + + Training computes ``moving_avr_dis_N`` on the post-transform but + UNSCALED discharge, then scales discharge. ``SapphirePredictor`` used + to compute the rolling means on SCALED discharge — feeding the model + past-covariates in a different magnitude band than it was trained on, + biasing predictions (NSE drop ≈ 0.08 in production). + + After the fix, calling ``SapphirePredictor.hindcast`` and + ``pipeline.hindcast.hindcast`` on the same data + checkpoint must + produce numerically equivalent Q50 values. + """ + from st_forecast import SapphirePredictor, train_model + from st_forecast.pipeline.hindcast import hindcast as pipeline_hindcast + + temporal_df, static_df = _synthetic_basin_data() + with tempfile.TemporaryDirectory(prefix="st_forecast_mavg_") as tmp: + work_dir = Path(tmp) + config = self._build_config_with_moving_avgs(work_dir) + train_model( + temporal_df=temporal_df, + static_df=static_df, + config=config, + output_path=work_dir, + train_years=[2020], + val_years=[2021], + seed=0, + ) + + target_code = 1002 + history_end = pd.Timestamp("2021-09-01") + cv_end = history_end + pd.Timedelta(days=config.forecast_horizon) + pm = temporal_df.loc[ + (temporal_df["code"] == target_code) + & (temporal_df["date"] <= history_end), + ["date", "code", "discharge"], + ].copy() + cv = temporal_df.loc[ + (temporal_df["code"] == target_code) + & (temporal_df["date"] <= cv_end), + ["date", "code", "T", "P"], + ].copy() + + sp_hc = SapphirePredictor(str(work_dir)).hindcast( + pm, cv, identifier=target_code, n=config.forecast_horizon + ) + pl_hc = pipeline_hindcast( + past_target=pm, + covariates=cv, + model_folder=work_dir, + n=config.forecast_horizon, + static_df=static_df, + codes=[target_code], + num_samples=config.num_samples, + ) + + # Align on (date, forecast_step) — same model, same data, must match. + join_keys = ["date", "forecast_step"] + merged = sp_hc.merge( + pl_hc, on=join_keys, suffixes=("_sp", "_pl") + ) + assert len(merged) > 0, "no overlapping rows between the two hindcasts" + + # Q50 spread is dominated by sampling noise (num_samples=20 with + # different RNG draws across the two paths). Assert mean + # absolute deviation is small relative to typical magnitudes. + mad = (merged["Q50_sp"] - merged["Q50_pl"]).abs().mean() + scale = merged["Q50_pl"].abs().mean() + assert mad < 0.25 * scale, ( + f"Q50 disagreement between SapphirePredictor and " + f"pipeline.hindcast is too large (MAD={mad:.3f}, " + f"scale={scale:.3f}); the train-vs-inference " + f"moving-average ordering bug may have regressed." + ) + + def test_hindcast_schema_matches_pipeline_hindcast(self): + """Regression: SapphirePredictor.hindcast must produce the same column + set as st_forecast.pipeline.hindcast.hindcast — downstream evaluation + in the SAPPHIRE forecast tools joins on `forecast_step` and + `forecast_issue_date`. Drift here silently collapses NSE because every + observation date matches `n` overlapping window-step rows. + """ + from st_forecast import SapphirePredictor, train_model + from st_forecast.pipeline.hindcast import hindcast as pipeline_hindcast + + temporal_df, static_df = _synthetic_basin_data() + with tempfile.TemporaryDirectory(prefix="st_forecast_hc_schema_") as tmp: + work_dir = Path(tmp) + config = self._build_config(work_dir) + train_model( + temporal_df=temporal_df, + static_df=static_df, + config=config, + output_path=work_dir, + train_years=[2020], + val_years=[2021], + seed=0, + ) + + target_code = 1002 + history_end = pd.Timestamp("2021-09-01") + cv_end = history_end + pd.Timedelta(days=config.forecast_horizon) + pm = temporal_df.loc[ + (temporal_df["code"] == target_code) + & (temporal_df["date"] <= history_end), + ["date", "code", "discharge"], + ].copy() + cv = temporal_df.loc[ + (temporal_df["code"] == target_code) + & (temporal_df["date"] <= cv_end), + ["date", "code", "T", "P"], + ].copy() + + sp_hc = SapphirePredictor(str(work_dir)).hindcast( + pm, cv, identifier=target_code, n=config.forecast_horizon + ) + pl_hc = pipeline_hindcast( + past_target=pm, + covariates=cv, + model_folder=work_dir, + n=config.forecast_horizon, + static_df=static_df, + codes=[target_code], + num_samples=config.num_samples, + ) + + assert set(sp_hc.columns) == set(pl_hc.columns), ( + f"Hindcast schemas drifted: only-SP={set(sp_hc.columns) - set(pl_hc.columns)}, " + f"only-pipeline={set(pl_hc.columns) - set(sp_hc.columns)}" + ) + for required in ("date", "code", "forecast_issue_date", "forecast_step"): + assert required in sp_hc.columns, f"missing column {required!r}" + + # forecast_step indexing within each window must be 1..n (no gaps, + # no shift) — this is what the user's downstream NSE-by-step joins on. + window = sp_hc[ + sp_hc["forecast_issue_date"] == sp_hc["forecast_issue_date"].iloc[0] + ].sort_values("date") + assert window["forecast_step"].tolist() == list( + range(1, config.forecast_horizon + 1) + ) + offsets = (window["date"] - window["forecast_issue_date"]).dt.days.tolist() + assert offsets == list(range(1, config.forecast_horizon + 1)), ( + f"date - forecast_issue_date offsets are not 1..n: {offsets}" + ) From 21fa297e73fafbf73d1f76c30f1e7af1130ed0b0 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Thu, 4 Jun 2026 09:14:03 +0200 Subject: [PATCH 10/20] kaz source added --- pyproject.toml | 4 + scripts/explore_kaz.py | 742 +++++++++++++++++++++++++ st_forecast/cli/assimilate.py | 31 +- st_forecast/cli/evaluate.py | 31 +- st_forecast/cli/predict.py | 32 +- st_forecast/cli/train.py | 32 +- st_forecast/cli/tune.py | 33 +- st_forecast/cli/visualize.py | 35 +- st_forecast/config/data_config.py | 51 +- st_forecast/data_utils/hive_loaders.py | 202 +++++++ st_forecast/data_utils/loaders.py | 161 +++++- st_forecast/data_utils/preparation.py | 28 +- st_forecast/get_data.py | 9 + st_forecast/pipeline/artifacts.py | 3 +- uv.lock | 510 +++++++++++++++++ 15 files changed, 1718 insertions(+), 186 deletions(-) create mode 100644 scripts/explore_kaz.py create mode 100644 st_forecast/data_utils/hive_loaders.py diff --git a/pyproject.toml b/pyproject.toml index df62832..6291057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,10 +61,14 @@ dev = [ "bump-my-version>=1.2.7", "cartopy>=0.25.0", "contextily>=1.7.0", + "folium>=0.20.0", "geopandas>=1.1.3", "matplotlib-scalebar>=0.9.0", + "plotly>=6.7.0", "pytest>=8.4.2", "ruff>=0.13.2", + "streamlit>=1.57.0", + "streamlit-folium>=0.27.2", ] [build-system] diff --git a/scripts/explore_kaz.py b/scripts/explore_kaz.py new file mode 100644 index 0000000..5b0fab5 --- /dev/null +++ b/scripts/explore_kaz.py @@ -0,0 +1,742 @@ +"""Interactive KAZ exploration dashboard. + +Launch with: + + uv run streamlit run scripts/explore_kaz.py + +Layout: +- Sidebar: dataset path, region, map-colour selector, aridity filter. +- Tab "Map": clickable folium map (polygons + gauge markers); right column shows + the selected basin's time series, runoff coefficients (under both unit + hypotheses), aridity, and streamflow histogram. +- Tab "Cross-basin": the area / aridity / specific-runoff / RR-histogram plots + for all gauged basins. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import branca.colormap as bcm +import folium +import geopandas as gpd +import numpy as np +import pandas as pd +import plotly.graph_objects as go +import streamlit as st +from plotly.subplots import make_subplots +from streamlit_folium import st_folium + +from st_forecast.data_utils.hive_loaders import ( + _attributes_file, + _timeseries_dir, + load_single_basin_timeseries, +) + +logging.basicConfig(level=logging.INFO, format="%(levelname)s | %(message)s") +logger = logging.getLogger(__name__) + +DEFAULT_BASE_PATH = ( + "/Users/sandrohunziker/Library/CloudStorage/OneDrive-hydrosolutionsltd/" + "KAZ_DATA/sandro-basins-kaz" +) +DEFAULT_REGION = "kaz" +VAR_MAP = { + "streamflow": "discharge", + "total_precipitation_sum": "P", + "temperature_2m_mean": "T", + "potential_evaporation_sum_FAO_PENMAN_MONTEITH": "PET", + "snow_depth_water_equivalent_mean": "SWE", +} +ZERO_FLOW_THRESHOLD = 0.001 # m³/s — anything below treated as "essentially dry" + + +# --------------------------------------------------------------------------- +# Data loading (all cached) +# --------------------------------------------------------------------------- + + +@st.cache_data(show_spinner="Loading gauge inventory…") +def load_gauge_inventory(base_path: str, region: str) -> pd.DataFrame: + attrs = pd.read_parquet(_attributes_file(base_path, region)) + keep = ["gauge_id", "gauge_lat", "gauge_lon", "area", "aridity_ERA5_LAND"] + attrs = attrs[keep].copy() + + stats: dict[str, dict[str, float]] = {} + for parquet in sorted( + _timeseries_dir(base_path, region).glob("gauge_id=*/data.parquet") + ): + gauge_id = parquet.parent.name.removeprefix("gauge_id=") + q = pd.read_parquet(parquet, columns=["streamflow"])["streamflow"] + n = int(q.notna().sum()) + stats[gauge_id] = { + "n_streamflow_obs": n, + "mean_q": float(q.mean()) if n else float("nan"), + "max_q": float(q.max()) if n else float("nan"), + } + summary = pd.DataFrame.from_dict(stats, orient="index").reset_index( + names="gauge_id" + ) + inv = attrs.merge(summary, on="gauge_id", how="left") + inv["n_streamflow_obs"] = inv["n_streamflow_obs"].fillna(0).astype(int) + inv["has_streamflow"] = inv["n_streamflow_obs"] > 0 + return inv + + +@st.cache_data(show_spinner="Loading basin polygons…") +def load_basin_shapes(base_path: str) -> gpd.GeoDataFrame: + shp = Path(base_path) / "updated_watersheds.shp" + gdf = gpd.read_file(shp).to_crs(4326) + gdf = gdf[["gauge_id", "geometry"]].copy() + # Shapefile uses "kazakhstan_"; the parquet attributes use "kaz_". + # Normalise so highlights and joins line up. + gdf["gauge_id"] = gdf["gauge_id"].str.replace(r"^kazakhstan_", "kaz_", regex=True) + gdf["geometry"] = gdf.geometry.simplify(0.005, preserve_topology=True) + return gdf + + +@st.cache_data(show_spinner=False) +def load_basin_ts(base_path: str, region: str, gauge_id: str) -> pd.DataFrame: + parquet = ( + _timeseries_dir(base_path, region) / f"gauge_id={gauge_id}" / "data.parquet" + ) + df = load_single_basin_timeseries(parquet, region, VAR_MAP) + return df.set_index("date").sort_index() + + +@st.cache_data(show_spinner="Computing water balance across all gauged basins…") +def compute_water_balance(base_path: str, region: str) -> pd.DataFrame: + inv = load_gauge_inventory(base_path, region) + rows: list[dict] = [] + for _, basin in inv[inv["has_streamflow"]].iterrows(): + gauge_id = basin["gauge_id"] + df = load_basin_ts(base_path, region, gauge_id).dropna(subset=["discharge"]) + if df.empty or pd.isna(basin["area"]): + continue + A = float(basin["area"]) + P_vol = float(df["P"].sum()) * A * 1000.0 + Q_sum = float(df["discharge"].sum()) + Q_vol_m3s = Q_sum * 86400.0 + Q_vol_mm = Q_sum * A * 1000.0 + spec_runoff = float(df["discharge"].mean()) * 86400.0 / (A * 1e6) * 1000.0 + rows.append( + { + "gauge_id": gauge_id, + "area_km2": A, + "aridity": float(basin["aridity_ERA5_LAND"]), + "n_days": int(len(df)), + "pct_zero_flow": float((df["discharge"] < ZERO_FLOW_THRESHOLD).mean()), + "mean_q_m3s": float(df["discharge"].mean()), + "p95_q_m3s": float(df["discharge"].quantile(0.95)), + "max_q_m3s": float(df["discharge"].max()), + "spec_runoff_mmday": spec_runoff, + "RR_m3s": Q_vol_m3s / P_vol if P_vol > 0 else float("nan"), + "RR_mmday": Q_vol_mm / P_vol if P_vol > 0 else float("nan"), + } + ) + return pd.DataFrame(rows) + + +def nearest_gauge(inv: pd.DataFrame, lat: float, lng: float) -> str: + d2 = (inv["gauge_lat"] - lat) ** 2 + (inv["gauge_lon"] - lng) ** 2 + return str(inv.loc[d2.idxmin(), "gauge_id"]) + + +# --------------------------------------------------------------------------- +# Map +# --------------------------------------------------------------------------- + + +def _safe_scale(values: pd.Series) -> tuple[float, float]: + v = values.replace([np.inf, -np.inf], np.nan).dropna() + if v.empty: + return 0.0, 1.0 + vmin, vmax = float(v.min()), float(v.max()) + if vmax <= vmin: + vmax = vmin + 1e-6 + return vmin, vmax + + +def build_map( + inv: pd.DataFrame, + wb: pd.DataFrame, + shapes: gpd.GeoDataFrame, + colour_by: str, + selected_gauge_id: str | None, +) -> folium.Map: + centre_lat = float(inv["gauge_lat"].mean()) + centre_lon = float(inv["gauge_lon"].mean()) + m = folium.Map( + location=[centre_lat, centre_lon], + zoom_start=5, + tiles="cartodbpositron", + control_scale=True, + ) + + folium.GeoJson( + shapes.to_json(), + name="Basin polygons", + style_function=lambda _: { + "fillColor": "#888888", + "fillOpacity": 0.08, + "color": "#555555", + "weight": 0.4, + }, + highlight_function=lambda _: {"weight": 1.2, "color": "#222"}, + ).add_to(m) + + # Overlay the selected basin's polygon in a bold contrasting colour + if selected_gauge_id is not None: + sel_shape = shapes[shapes["gauge_id"] == selected_gauge_id] + if not sel_shape.empty: + folium.GeoJson( + sel_shape.to_json(), + name="Selected basin", + style_function=lambda _: { + "fillColor": "#e74c3c", + "fillOpacity": 0.20, + "color": "#c0392b", + "weight": 4.0, + "opacity": 1.0, + }, + ).add_to(m) + + # Merge runoff stats onto gauged rows so colour-by can use them + gauged = inv[inv["has_streamflow"]].merge( + wb[["gauge_id", "RR_m3s", "spec_runoff_mmday", "pct_zero_flow"]], + on="gauge_id", + how="left", + ) + ungauged = inv[~inv["has_streamflow"]] + + colour_field = { + "Aridity (ERA5-Land)": "aridity_ERA5_LAND", + "Runoff ratio (m³/s)": "RR_m3s", + "Specific runoff (mm/day)": "spec_runoff_mmday", + "# streamflow obs": "n_streamflow_obs", + }[colour_by] + + log_scale = colour_field in {"RR_m3s", "spec_runoff_mmday"} + raw = gauged[colour_field] + if log_scale: + positives = raw[raw > 0] + log_vals = np.log10(positives) if not positives.empty else pd.Series([0.0]) + vmin, vmax = _safe_scale(log_vals) + colormap = bcm.linear.YlOrRd_09.scale(vmin, vmax) + colormap.caption = f"log10 {colour_by}" + else: + vmin, vmax = _safe_scale(raw) + colormap = bcm.linear.YlOrRd_09.scale(vmin, vmax) + colormap.caption = colour_by + + for _, row in ungauged.iterrows(): + folium.CircleMarker( + location=[row["gauge_lat"], row["gauge_lon"]], + radius=3, + color="#888", + weight=0.6, + fill=True, + fillColor="#ddd", + fillOpacity=0.6, + tooltip=f"{row['gauge_id']} — no streamflow", + ).add_to(m) + + for _, row in gauged.iterrows(): + v = row[colour_field] + if pd.isna(v) or (log_scale and v <= 0): + colour = "#aaaaaa" + else: + colour = colormap(np.log10(v) if log_scale else v) + is_selected = selected_gauge_id == row["gauge_id"] + rr_disp = row["RR_m3s"] + rr_text = "n/a" if pd.isna(rr_disp) else f"{rr_disp:.3g}" + tooltip = ( + f"{row['gauge_id']}
" + f"area = {row['area']:.0f} km²
" + f"aridity = {row['aridity_ERA5_LAND']:.2f}
" + f"n_obs = {row['n_streamflow_obs']}
" + f"RR (m³/s) = {rr_text}" + ) + folium.CircleMarker( + location=[row["gauge_lat"], row["gauge_lon"]], + radius=10 if is_selected else 7, + color="#111" if is_selected else "#222", + weight=2.5 if is_selected else 0.8, + fill=True, + fillColor=colour, + fillOpacity=0.95, + tooltip=folium.Tooltip(tooltip, sticky=True), + ).add_to(m) + + colormap.add_to(m) + return m + + +# --------------------------------------------------------------------------- +# Plotly figures +# --------------------------------------------------------------------------- + + +def timeseries_figure(df: pd.DataFrame, gauge_id: str) -> go.Figure: + fig = make_subplots( + rows=2, + cols=1, + shared_xaxes=True, + row_heights=[0.35, 0.65], + vertical_spacing=0.04, + ) + fig.add_trace( + go.Bar( + x=df.index, + y=df["P"], + name="Precipitation", + marker_color="#3a78b4", + opacity=0.85, + ), + row=1, + col=1, + ) + fig.add_trace( + go.Scatter( + x=df.index, + y=df["discharge"], + name="Streamflow", + line=dict(color="#111111", width=0.9), + ), + row=2, + col=1, + ) + # Flip precipitation axis so bars hang from the top + fig.update_yaxes(autorange="reversed", title_text="P (mm/day)", row=1, col=1) + fig.update_yaxes(title_text="Q (m³/s)", row=2, col=1) + fig.update_xaxes(title_text="Date", row=2, col=1) + fig.update_layout( + height=420, + margin=dict(l=50, r=30, t=40, b=40), + title=f"{gauge_id} — daily precipitation & streamflow", + showlegend=False, + hovermode="x unified", + ) + return fig + + +def streamflow_histogram(df: pd.DataFrame) -> go.Figure: + q = df["discharge"].dropna() + positives = q[q > 0] + if positives.empty: + fig = go.Figure() + fig.update_layout(title="No positive streamflow", height=260) + return fig + fig = go.Figure() + fig.add_trace( + go.Histogram( + x=np.log10(positives), + nbinsx=40, + marker_color="#1f77b4", + opacity=0.85, + ) + ) + fig.update_layout( + title="Streamflow distribution", + xaxis_title="log10(Q [m³/s])", + yaxis_title="Days", + height=260, + margin=dict(l=40, r=20, t=40, b=40), + bargap=0.05, + ) + return fig + + +def cross_basin_figure(wb: pd.DataFrame) -> go.Figure: + fig = make_subplots( + rows=2, + cols=2, + subplot_titles=( + "Area vs mean streamflow", + "Aridity vs mean streamflow", + "Aridity vs specific runoff (Q/A)", + "Runoff ratio histograms", + ), + horizontal_spacing=0.10, + vertical_spacing=0.13, + ) + + g = wb.dropna(subset=["area_km2", "mean_q_m3s", "aridity"]).copy() + log_area = np.log10(g["area_km2"]) + + # (1) Area vs mean Q, colour = aridity + fig.add_trace( + go.Scatter( + x=g["area_km2"], + y=g["mean_q_m3s"], + mode="markers", + marker=dict( + size=8, + color=g["aridity"], + colorscale="YlOrBr", + colorbar=dict(title="Aridity", x=0.46, y=0.78, len=0.36), + line=dict(color="black", width=0.4), + ), + text=g["gauge_id"], + hovertemplate="%{text}
area=%{x:.0f} km²
Q̄=%{y:.3g} m³/s", + name="basins", + ), + row=1, + col=1, + ) + r_area = float(np.corrcoef(log_area, np.log10(g["mean_q_m3s"]))[0, 1]) + fig.add_annotation( + text=f"corr(log A, log Q̄) = {r_area:.2f}", + xref="x domain", + yref="y domain", + x=0.04, + y=0.96, + showarrow=False, + row=1, + col=1, + bgcolor="rgba(255,255,255,0.85)", + bordercolor="#bbb", + borderwidth=1, + font=dict(size=10), + ) + + # (2) Aridity vs mean Q, colour = log(area) + fig.add_trace( + go.Scatter( + x=g["aridity"], + y=g["mean_q_m3s"], + mode="markers", + marker=dict( + size=8, + color=log_area, + colorscale="Viridis", + colorbar=dict(title="log10 A (km²)", x=1.02, y=0.78, len=0.36), + line=dict(color="black", width=0.4), + ), + text=g["gauge_id"], + hovertemplate="%{text}
aridity=%{x:.2f}
Q̄=%{y:.3g} m³/s", + showlegend=False, + ), + row=1, + col=2, + ) + r_arid = float(np.corrcoef(g["aridity"], np.log10(g["mean_q_m3s"]))[0, 1]) + fig.add_annotation( + text=f"corr(aridity, log Q̄) = {r_arid:.2f}", + xref="x2 domain", + yref="y2 domain", + x=0.04, + y=0.96, + showarrow=False, + row=1, + col=2, + bgcolor="rgba(255,255,255,0.85)", + bordercolor="#bbb", + borderwidth=1, + font=dict(size=10), + ) + + # (3) Aridity vs specific runoff + g3 = g[g["spec_runoff_mmday"] > 0].copy() + fig.add_trace( + go.Scatter( + x=g3["aridity"], + y=g3["spec_runoff_mmday"], + mode="markers", + marker=dict( + size=8, + color=np.log10(g3["area_km2"]), + colorscale="Viridis", + showscale=False, + line=dict(color="black", width=0.4), + ), + text=g3["gauge_id"], + hovertemplate="%{text}
aridity=%{x:.2f}
Q/A=%{y:.3g} mm/day", + showlegend=False, + ), + row=2, + col=1, + ) + r_spec = float(np.corrcoef(g3["aridity"], np.log10(g3["spec_runoff_mmday"]))[0, 1]) + fig.add_annotation( + text=f"corr(aridity, log Q/A) = {r_spec:.2f}", + xref="x3 domain", + yref="y3 domain", + x=0.04, + y=0.96, + showarrow=False, + row=2, + col=1, + bgcolor="rgba(255,255,255,0.85)", + bordercolor="#bbb", + borderwidth=1, + font=dict(size=10), + ) + + # (4) RR histograms + floor = 10**-5 + + def _rr_clip(s: pd.Series) -> pd.Series: + return s.replace([np.inf, -np.inf], np.nan).dropna().clip(lower=floor) + + rr_m3s = _rr_clip(wb["RR_m3s"]) + rr_mm = _rr_clip(wb["RR_mmday"]) + fig.add_trace( + go.Histogram( + x=np.log10(rr_m3s), + xbins=dict(start=-5, end=1, size=0.25), + name="RR (m³/s)", + marker_color="#1f77b4", + opacity=0.7, + ), + row=2, + col=2, + ) + fig.add_trace( + go.Histogram( + x=np.log10(rr_mm), + xbins=dict(start=-5, end=1, size=0.25), + name="RR (mm/day)", + marker_color="#d62728", + opacity=0.6, + ), + row=2, + col=2, + ) + fig.add_vline( + x=0.0, + line=dict(color="#c0392b", dash="dash", width=1.5), + annotation_text="RR = 1 (Q = P)", + annotation_position="top", + row=2, + col=2, + ) + + # Axes + fig.update_xaxes(type="log", title_text="Area (km²)", row=1, col=1) + fig.update_yaxes(type="log", title_text="Mean Q (m³/s)", row=1, col=1) + fig.update_xaxes(title_text="Aridity index", row=1, col=2) + fig.update_yaxes(type="log", title_text="Mean Q (m³/s)", row=1, col=2) + fig.update_xaxes(title_text="Aridity index", row=2, col=1) + fig.update_yaxes(type="log", title_text="Q/A (mm/day)", row=2, col=1) + fig.update_xaxes(title_text="log10 RR", row=2, col=2) + fig.update_yaxes(title_text="# basins", row=2, col=2) + + n_m3s_impossible = int((wb["RR_m3s"] > 1).sum()) + n_mm_impossible = int((wb["RR_mmday"] > 1).sum()) + fig.update_layout( + height=720, + margin=dict(l=50, r=120, t=60, b=50), + barmode="overlay", + title=( + f"Cross-basin diagnostics (n={len(wb)} gauged basins) — " + f"basins with RR>1: m³/s={n_m3s_impossible}, mm/day={n_mm_impossible}" + ), + legend=dict(x=0.78, y=0.45), + ) + return fig + + +# --------------------------------------------------------------------------- +# Streamlit app +# --------------------------------------------------------------------------- + + +def render_basin_panel( + base_path: str, + region: str, + inv: pd.DataFrame, + wb: pd.DataFrame, + gauge_id: str, +) -> None: + basin = inv.loc[inv["gauge_id"] == gauge_id].iloc[0] + wb_row = wb.loc[wb["gauge_id"] == gauge_id] + has_wb = not wb_row.empty + wb_row = wb_row.iloc[0] if has_wb else None + + st.markdown(f"### {gauge_id}") + cols = st.columns(4) + cols[0].metric("Area", f"{basin['area']:.0f} km²") + cols[1].metric("Aridity", f"{basin['aridity_ERA5_LAND']:.2f}") + cols[2].metric("# obs", f"{int(basin['n_streamflow_obs'])}") + cols[3].metric( + "% zero-flow days", + f"{wb_row['pct_zero_flow'] * 100:.1f}%" if has_wb else "—", + ) + + if not basin["has_streamflow"]: + st.info("This gauge has no streamflow observations.") + return + + ts = load_basin_ts(base_path, region, gauge_id) + ts_plot = ts[ts.index >= pd.Timestamp("2000-01-01")] + if ts_plot.empty: + ts_plot = ts + st.plotly_chart( + timeseries_figure(ts_plot, gauge_id), + use_container_width=True, + ) + + cols_rr = st.columns(2) + if has_wb: + rr_m3s = wb_row["RR_m3s"] + rr_mm = wb_row["RR_mmday"] + cols_rr[0].metric( + "Runoff ratio — Q in m³/s", + f"{rr_m3s:.3g}", + help="ΣQ·86400 / ΣP·A·1000 — physically plausible if ≤ ~1.", + ) + cols_rr[1].metric( + "Runoff ratio — Q in mm/day", + f"{rr_mm:.3g}", + help="ΣQ·A·1000 / ΣP·A·1000 = ΣQ/ΣP — physically plausible if ≤ ~1.", + ) + flags: list[str] = [] + if rr_m3s > 1.0: + flags.append( + f"RR_m3s = {rr_m3s:.2g} > 1: Q > P (glacier melt / ERA5 undercatch)" + ) + if rr_mm > 1.0: + flags.append( + f"RR_mmday = {rr_mm:.2g} > 1: Q > P → mm/day unit hypothesis violated" + ) + if rr_m3s < 0.005: + flags.append( + f"RR_m3s = {rr_m3s:.2g}: extremely low — likely endorheic, irrigation losses, " + f"or topographic area > effective contributing area" + ) + if flags: + st.warning(" • " + "\n\n • ".join(flags)) + + st.plotly_chart(streamflow_histogram(ts), use_container_width=True) + + +def main() -> None: + st.set_page_config(page_title="KAZ explorer", layout="wide") + st.title("Kazakhstan basin explorer") + + with st.sidebar: + st.markdown("### Dataset") + base_path = st.text_input("Hive base path", value=DEFAULT_BASE_PATH) + region = st.text_input("Region", value=DEFAULT_REGION) + if not Path(base_path).exists(): + st.error(f"Path not found: {base_path}") + st.stop() + + st.markdown("### Map") + colour_by = st.selectbox( + "Colour gauges by", + [ + "Aridity (ERA5-Land)", + "Runoff ratio (m³/s)", + "Specific runoff (mm/day)", + "# streamflow obs", + ], + index=0, + ) + + inv = load_gauge_inventory(base_path, region) + shapes = load_basin_shapes(base_path) + wb = compute_water_balance(base_path, region) + + with st.sidebar: + st.markdown("### Filters") + arid_min, arid_max = ( + float(inv["aridity_ERA5_LAND"].min()), + float(inv["aridity_ERA5_LAND"].max()), + ) + arid_range = st.slider( + "Aridity range", + min_value=round(arid_min, 2), + max_value=round(arid_max, 2), + value=(round(arid_min, 2), round(arid_max, 2)), + step=0.1, + ) + only_gauged = st.checkbox("Only basins with streamflow", value=False) + st.caption( + f"{int(inv['has_streamflow'].sum())} gauged of {len(inv)} basins. " + f"Water balance computed for {len(wb)} basins." + ) + + inv_filt = inv[inv["aridity_ERA5_LAND"].between(arid_range[0], arid_range[1])] + if only_gauged: + inv_filt = inv_filt[inv_filt["has_streamflow"]] + + if inv_filt.empty: + st.warning("No basins match the current filters — widen the aridity range.") + st.stop() + + if ( + "selected_gauge" not in st.session_state + or st.session_state.selected_gauge not in set(inv_filt["gauge_id"]) + ): + gauged_filt = inv_filt[inv_filt["has_streamflow"]].sort_values( + "n_streamflow_obs", ascending=False + ) + default = ( + gauged_filt.iloc[0]["gauge_id"] + if not gauged_filt.empty + else inv_filt.iloc[0]["gauge_id"] + ) + st.session_state.selected_gauge = default + + st.caption( + "Tab 1 → click a basin on the map to inspect it. " + "Tab 2 → all-basin correlation plots and runoff-ratio distributions." + ) + tab_map, tab_cross = st.tabs( + ["🗺️ Map & selected basin", "📊 Correlations & distributions (all basins)"] + ) + + with tab_map: + left, right = st.columns([0.55, 0.45]) + + with left: + m = build_map( + inv_filt, wb, shapes, colour_by, st.session_state.selected_gauge + ) + map_result = st_folium( + m, + width=None, + height=620, + returned_objects=["last_object_clicked"], + key="kaz_map", + ) + + sel_options = inv_filt.sort_values( + ["has_streamflow", "n_streamflow_obs"], ascending=[False, False] + )["gauge_id"].tolist() + current = st.session_state.selected_gauge + if current not in sel_options and sel_options: + current = sel_options[0] + sel = st.selectbox( + "…or pick a gauge from the list", + options=sel_options, + index=sel_options.index(current) if current in sel_options else 0, + key="gauge_selectbox", + ) + if sel != st.session_state.selected_gauge: + st.session_state.selected_gauge = sel + st.rerun() + + if map_result and map_result.get("last_object_clicked"): + click = map_result["last_object_clicked"] + clicked = nearest_gauge(inv_filt, click["lat"], click["lng"]) + if clicked != st.session_state.selected_gauge: + st.session_state.selected_gauge = clicked + st.rerun() + + with right: + render_basin_panel( + base_path, region, inv, wb, st.session_state.selected_gauge + ) + + with tab_cross: + st.plotly_chart(cross_basin_figure(wb), use_container_width=True) + with st.expander("Show water-balance table"): + st.dataframe(wb.sort_values("RR_m3s"), use_container_width=True) + + +if __name__ == "__main__": + main() diff --git a/st_forecast/cli/assimilate.py b/st_forecast/cli/assimilate.py index dc29ed5..036ddd6 100644 --- a/st_forecast/cli/assimilate.py +++ b/st_forecast/cli/assimilate.py @@ -9,7 +9,7 @@ warnings.filterwarnings("ignore", category=UserWarning, module="fs") -from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.data_utils.loaders import build_collector from st_forecast.pipeline.artifacts import load_run_artifacts from st_forecast.pipeline.assimilation import assimilate_hindcast @@ -94,34 +94,7 @@ def assimilate_cli( click.echo("Loading observation data from config paths...") # Route to appropriate collector based on data_source - if getattr(run_config, "data_source", "legacy") == "caravan" and run_config.caravan: - from st_forecast.config.data_config import CaravanConfig - - caravan_config = CaravanConfig( - base_path=run_config.caravan["base_path"], - region=run_config.caravan["region"], - n_basins=run_config.caravan.get("n_basins"), - basin_ids=run_config.caravan.get("basin_ids"), - start_date=run_config.caravan.get("start_date"), - end_date=run_config.caravan.get("end_date"), - ) - collector = CaravanDataCollector(caravan_config) - click.echo(f"Using Caravan data source: {caravan_config.region}") - else: - paths_dict = { - "path_rivers": run_config.path_rivers, - "path_forcing": run_config.path_forcing, - "path_to_operational_forcing": run_config.path_to_operational_forcing, - "path_to_hindcast_forcing": run_config.path_to_hindcast_forcing, - "HRU_forcing": run_config.HRU_forcing, - "path_static": run_config.path_static, - "path_to_sla": run_config.path_to_sla, - "path_to_nir": run_config.path_to_nir, - "path_to_swe": run_config.path_to_swe, - "path_to_hs": run_config.path_to_hs, - "path_to_rof": run_config.path_to_rof, - } - collector = RawDataCollector(paths_dict, run_config.region) + collector = build_collector(run_config) raw_data = collector.collect_all() observations_df = raw_data["discharge"].rename(columns={run_config.target_col: "Q"}) diff --git a/st_forecast/cli/evaluate.py b/st_forecast/cli/evaluate.py index 68a7df0..c7d75e3 100644 --- a/st_forecast/cli/evaluate.py +++ b/st_forecast/cli/evaluate.py @@ -6,7 +6,7 @@ import click import pandas as pd -from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.data_utils.loaders import build_collector from st_forecast.evaluation.evaluator import EvaluationConfig, evaluate from st_forecast.pipeline.artifacts import load_run_artifacts @@ -45,34 +45,7 @@ def evaluate_cli(run_folder: Path, split: str, agricultural_only: bool) -> None: logger.info(f"Loaded {len(predictions_df)} prediction rows from {predictions_path}") # 3. Load observations - route to appropriate collector based on data_source - if getattr(run_config, "data_source", "legacy") == "caravan" and run_config.caravan: - from st_forecast.config.data_config import CaravanConfig - - caravan_config = CaravanConfig( - base_path=run_config.caravan["base_path"], - region=run_config.caravan["region"], - n_basins=run_config.caravan.get("n_basins"), - basin_ids=run_config.caravan.get("basin_ids"), - start_date=run_config.caravan.get("start_date"), - end_date=run_config.caravan.get("end_date"), - ) - collector = CaravanDataCollector(caravan_config) - logger.info(f"Using Caravan data source: {caravan_config.region}") - else: - paths_dict = { - "path_rivers": run_config.path_rivers, - "path_forcing": run_config.path_forcing, - "path_to_operational_forcing": run_config.path_to_operational_forcing, - "path_to_hindcast_forcing": run_config.path_to_hindcast_forcing, - "HRU_forcing": run_config.HRU_forcing, - "path_static": run_config.path_static, - "path_to_sla": run_config.path_to_sla, - "path_to_nir": run_config.path_to_nir, - "path_to_swe": run_config.path_to_swe, - "path_to_hs": run_config.path_to_hs, - "path_to_rof": run_config.path_to_rof, - } - collector = RawDataCollector(paths_dict, run_config.region) + collector = build_collector(run_config) observations_df = collector.collect_discharge() logger.info(f"Loaded {len(observations_df)} observation rows") diff --git a/st_forecast/cli/predict.py b/st_forecast/cli/predict.py index 7cc759a..2ad34bc 100644 --- a/st_forecast/cli/predict.py +++ b/st_forecast/cli/predict.py @@ -10,7 +10,7 @@ warnings.filterwarnings("ignore", category=UserWarning, module="fs") from st_forecast import get_data -from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.data_utils.loaders import build_collector from st_forecast.data_utils.preparation import ( extract_timeseries_components, prepare_validation_data, @@ -83,35 +83,7 @@ def predict_cli( ) # Load raw data - route to appropriate collector based on data_source - if getattr(run_config, "data_source", "legacy") == "caravan" and run_config.caravan: - from st_forecast.config.data_config import CaravanConfig - - caravan_config = CaravanConfig( - base_path=run_config.caravan["base_path"], - region=run_config.caravan["region"], - n_basins=run_config.caravan.get("n_basins"), - basin_ids=run_config.caravan.get("basin_ids"), - start_date=run_config.caravan.get("start_date"), - end_date=run_config.caravan.get("end_date"), - ) - collector = CaravanDataCollector(caravan_config) - logger.info(f"Using Caravan data source: {caravan_config.region}") - else: - # Build paths dict for RawDataCollector - paths_dict = { - "path_rivers": run_config.path_rivers, - "path_forcing": run_config.path_forcing, - "path_to_operational_forcing": run_config.path_to_operational_forcing, - "path_to_hindcast_forcing": run_config.path_to_hindcast_forcing, - "HRU_forcing": run_config.HRU_forcing, - "path_static": run_config.path_static, - "path_to_sla": run_config.path_to_sla, - "path_to_nir": run_config.path_to_nir, - "path_to_swe": run_config.path_to_swe, - "path_to_hs": run_config.path_to_hs, - "path_to_rof": run_config.path_to_rof, - } - collector = RawDataCollector(paths_dict, run_config.region) + collector = build_collector(run_config) raw_data = collector.collect_all() logger.info("Raw data collected") diff --git a/st_forecast/cli/train.py b/st_forecast/cli/train.py index 15bf600..3a65a03 100644 --- a/st_forecast/cli/train.py +++ b/st_forecast/cli/train.py @@ -11,7 +11,7 @@ from pathlib import Path from st_forecast import get_data -from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.data_utils.loaders import build_collector from st_forecast.data_utils.preparation import ( extract_timeseries_components, prepare_validation_data, @@ -85,35 +85,7 @@ def train_cli(run_folder: Path, config: Path | None, seed: int) -> None: logger.info(f"Loaded config for region: {run_config.region}") # Load raw data - route to appropriate collector based on data_source - if getattr(run_config, "data_source", "legacy") == "caravan" and run_config.caravan: - from st_forecast.config.data_config import CaravanConfig - - caravan_config = CaravanConfig( - base_path=run_config.caravan["base_path"], - region=run_config.caravan["region"], - n_basins=run_config.caravan.get("n_basins"), - basin_ids=run_config.caravan.get("basin_ids"), - start_date=run_config.caravan.get("start_date"), - end_date=run_config.caravan.get("end_date"), - ) - collector = CaravanDataCollector(caravan_config) - logger.info(f"Using Caravan data source: {caravan_config.region}") - else: - # Build paths dict for RawDataCollector - paths_dict = { - "path_rivers": run_config.path_rivers, - "path_forcing": run_config.path_forcing, - "path_to_operational_forcing": run_config.path_to_operational_forcing, - "path_to_hindcast_forcing": run_config.path_to_hindcast_forcing, - "HRU_forcing": run_config.HRU_forcing, - "path_static": run_config.path_static, - "path_to_sla": run_config.path_to_sla, - "path_to_nir": run_config.path_to_nir, - "path_to_swe": run_config.path_to_swe, - "path_to_hs": run_config.path_to_hs, - "path_to_rof": run_config.path_to_rof, - } - collector = RawDataCollector(paths_dict, run_config.region) + collector = build_collector(run_config) raw_data = collector.collect_all() logger.info("Raw data collected") diff --git a/st_forecast/cli/tune.py b/st_forecast/cli/tune.py index 3ff62c9..1e2972b 100644 --- a/st_forecast/cli/tune.py +++ b/st_forecast/cli/tune.py @@ -11,7 +11,7 @@ import pandas as pd from st_forecast import get_data -from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.data_utils.loaders import build_collector from st_forecast.data_utils.preparation import ( extract_timeseries_components, prepare_validation_data, @@ -30,35 +30,8 @@ def _load_and_prepare_data( config: RunConfig, ) -> tuple[dict, pd.DataFrame, dict]: # Route to appropriate collector based on data_source - if getattr(config, "data_source", "legacy") == "caravan" and config.caravan: - from st_forecast.config.data_config import CaravanConfig - - caravan_config = CaravanConfig( - base_path=config.caravan["base_path"], - region=config.caravan["region"], - n_basins=config.caravan.get("n_basins"), - basin_ids=config.caravan.get("basin_ids"), - start_date=config.caravan.get("start_date"), - end_date=config.caravan.get("end_date"), - ) - collector = CaravanDataCollector(caravan_config) - logger.info(f"Using Caravan data source: {caravan_config.region}") - paths_dict = {} # Not used for Caravan - else: - paths_dict = { - "path_rivers": config.path_rivers, - "path_forcing": config.path_forcing, - "path_to_operational_forcing": config.path_to_operational_forcing, - "path_to_hindcast_forcing": config.path_to_hindcast_forcing, - "HRU_forcing": config.HRU_forcing, - "path_static": config.path_static, - "path_to_sla": config.path_to_sla, - "path_to_nir": config.path_to_nir, - "path_to_swe": config.path_to_swe, - "path_to_hs": config.path_to_hs, - "path_to_rof": config.path_to_rof, - } - collector = RawDataCollector(paths_dict, config.region) + collector = build_collector(config) + paths_dict: dict = {} raw_data = collector.collect_all() logger.info("Raw data collected") diff --git a/st_forecast/cli/visualize.py b/st_forecast/cli/visualize.py index 80e218a..14106a7 100644 --- a/st_forecast/cli/visualize.py +++ b/st_forecast/cli/visualize.py @@ -3,7 +3,7 @@ import click -from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.data_utils.loaders import build_collector from st_forecast.pipeline.artifacts import load_run_artifacts from st_forecast.visualization import VisualizationConfig, generate_visualizations @@ -118,38 +118,7 @@ def visualize_cli( observations_df = None if "forecasts" in plot_types or "all" in plot_types: # Route to appropriate collector based on data_source - if ( - getattr(run_config, "data_source", "legacy") == "caravan" - and run_config.caravan - ): - from st_forecast.config.data_config import CaravanConfig - - caravan_config = CaravanConfig( - base_path=run_config.caravan["base_path"], - region=run_config.caravan["region"], - n_basins=run_config.caravan.get("n_basins"), - basin_ids=run_config.caravan.get("basin_ids"), - start_date=run_config.caravan.get("start_date"), - end_date=run_config.caravan.get("end_date"), - ) - collector = CaravanDataCollector(caravan_config) - logger.info(f"Using Caravan data source: {caravan_config.region}") - else: - paths_dict = { - "path_rivers": run_config.path_rivers, - "path_forcing": run_config.path_forcing, - "path_to_operational_forcing": run_config.path_to_operational_forcing, - "path_to_hindcast_forcing": run_config.path_to_hindcast_forcing, - "HRU_forcing": run_config.HRU_forcing, - "path_static": run_config.path_static, - "path_to_sla": run_config.path_to_sla, - "path_to_nir": run_config.path_to_nir, - "path_to_swe": run_config.path_to_swe, - "path_to_hs": run_config.path_to_hs, - "path_to_rof": run_config.path_to_rof, - } - collector = RawDataCollector(paths_dict, run_config.region) - + collector = build_collector(run_config) observations_df = collector.collect_discharge() logger.info( f"Loaded {len(observations_df)} observation rows for forecast plots" diff --git a/st_forecast/config/data_config.py b/st_forecast/config/data_config.py index 5c2954c..552c72d 100644 --- a/st_forecast/config/data_config.py +++ b/st_forecast/config/data_config.py @@ -162,6 +162,33 @@ class CaravanConfig: ) +@dataclass +class HiveConfig: + """Configuration for loading data from a Hive-partitioned parquet dataset. + + Expected layout: + {base_path}/REGION_NAME={region}/data_type=timeseries/temporal_resolution=daily/gauge_id={region}_{basin_id}/data.parquet + {base_path}/REGION_NAME={region}/data_type=attributes/data.parquet + """ + + base_path: str + region: str + n_basins: int | None = None + basin_ids: list[str] | None = None + start_date: str | None = None + end_date: str | None = None + drop_empty_discharge: bool = True + variable_mapping: dict[str, str] = field( + default_factory=lambda: { + "streamflow": "discharge", + "total_precipitation_sum": "P", + "temperature_2m_mean": "T", + "potential_evaporation_sum_FAO_PENMAN_MONTEITH": "PET", + "snow_depth_water_equivalent_mean": "SWE", + } + ) + + @dataclass class DataConfig: """Master configuration object for the entire data pipeline. @@ -174,8 +201,9 @@ class DataConfig: country: str | None = None # Data source selection - data_source: str = "legacy" # 'legacy' or 'caravan' + data_source: str = "legacy" # 'legacy', 'caravan', or 'hive' caravan: CaravanConfig | None = None + hive: HiveConfig | None = None # Sub-configurations paths: PathConfig = field(default_factory=PathConfig) @@ -219,6 +247,26 @@ def from_dict(cls, config_dict: dict[str, Any]) -> "DataConfig": ), ) + # Extract Hive config if present + hive_config = None + if "hive" in config_dict and config_dict["hive"] is not None: + hive_dict = config_dict["hive"] + hive_config = HiveConfig( + base_path=hive_dict["base_path"], + region=hive_dict["region"], + n_basins=hive_dict.get("n_basins"), + basin_ids=hive_dict.get("basin_ids"), + start_date=hive_dict.get("start_date"), + end_date=hive_dict.get("end_date"), + drop_empty_discharge=hive_dict.get("drop_empty_discharge", True), + variable_mapping=hive_dict.get( + "variable_mapping", + HiveConfig.__dataclass_fields__[ + "variable_mapping" + ].default_factory(), + ), + ) + # Extract path-related keys path_keys = [ "path_rivers", @@ -331,6 +379,7 @@ def from_dict(cls, config_dict: dict[str, Any]) -> "DataConfig": country=config_dict.get("country"), data_source=config_dict.get("data_source", "legacy"), caravan=caravan_config, + hive=hive_config, paths=paths, temporal_split=temporal_split, features=features, diff --git a/st_forecast/data_utils/hive_loaders.py b/st_forecast/data_utils/hive_loaders.py new file mode 100644 index 0000000..86fbc57 --- /dev/null +++ b/st_forecast/data_utils/hive_loaders.py @@ -0,0 +1,202 @@ +"""Data loading functions for Hive-partitioned parquet datasets. + +Layout expected: +- {base_path}/REGION_NAME={region}/data_type=timeseries/temporal_resolution=daily/gauge_id={region}_{basin_id}/data.parquet +- {base_path}/REGION_NAME={region}/data_type=attributes/data.parquet +""" + +import logging +from pathlib import Path + +import pandas as pd + +logger = logging.getLogger(__name__) + + +def _region_root(base_path: str, region: str) -> Path: + return Path(base_path) / f"REGION_NAME={region}" + + +def _timeseries_dir(base_path: str, region: str) -> Path: + return ( + _region_root(base_path, region) + / "data_type=timeseries" + / "temporal_resolution=daily" + ) + + +def _attributes_file(base_path: str, region: str) -> Path: + return _region_root(base_path, region) / "data_type=attributes" / "data.parquet" + + +def _basin_id_from_path(parquet_path: Path, region: str) -> str: + parent = parquet_path.parent.name + prefix = "gauge_id=" + if not parent.startswith(prefix): + raise ValueError(f"Unexpected partition folder: {parent}") + gauge_id = parent[len(prefix) :] + region_prefix = f"{region}_" + return gauge_id[len(region_prefix) :] if gauge_id.startswith(region_prefix) else gauge_id + + +def _to_int_or_str(value: str) -> int | str: + try: + return int(value) + except ValueError: + return value + + +def discover_basin_files( + base_path: str, + region: str, + basin_ids: list[str] | None = None, +) -> list[Path]: + ts_dir = _timeseries_dir(base_path, region) + if not ts_dir.exists(): + raise FileNotFoundError(f"Timeseries directory not found: {ts_dir}") + + all_files = sorted(ts_dir.glob("gauge_id=*/data.parquet")) + if not all_files: + raise FileNotFoundError(f"No parquet files found under {ts_dir}") + + if basin_ids is not None: + wanted = {str(b) for b in basin_ids} + selected = [ + f for f in all_files if _basin_id_from_path(f, region) in wanted + ] + missing = wanted - {_basin_id_from_path(f, region) for f in selected} + for m in missing: + logger.warning(f"Basin not found in hive layout: {region}_{m}") + return selected + + return all_files + + +def load_single_basin_timeseries( + parquet_path: Path, + region: str, + variable_mapping: dict[str, str], + start_date: str | None = None, + end_date: str | None = None, +) -> pd.DataFrame: + df = pd.read_parquet(parquet_path) + basin_id = _to_int_or_str(_basin_id_from_path(parquet_path, region)) + df["code"] = basin_id + + rename_map = {k: v for k, v in variable_mapping.items() if k in df.columns} + df = df.rename(columns=rename_map) + + df["date"] = pd.to_datetime(df["date"]) + if start_date is not None: + df = df[df["date"] >= pd.Timestamp(start_date)] + if end_date is not None: + df = df[df["date"] <= pd.Timestamp(end_date)] + + keep_cols = ["date", "code"] + [v for v in variable_mapping.values() if v in df.columns] + return df[keep_cols].copy() + + +def load_hive_timeseries( + base_path: str, + region: str, + variable_mapping: dict[str, str], + n_basins: int | None = None, + basin_ids: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, + drop_empty_discharge: bool = True, +) -> pd.DataFrame: + files = discover_basin_files(base_path, region, basin_ids) + + dfs: list[pd.DataFrame] = [] + dropped_empty: list[int | str] = [] + for path in files: + if n_basins is not None and len(dfs) >= n_basins: + break + try: + df = load_single_basin_timeseries( + path, region, variable_mapping, start_date, end_date + ) + except Exception as e: + logger.warning(f"Failed to load {path}: {e}") + continue + + if drop_empty_discharge and "discharge" in df.columns: + if df["discharge"].notna().sum() == 0: + dropped_empty.append(df["code"].iloc[0] if len(df) else "?") + continue + dfs.append(df) + + if dropped_empty: + logger.info( + f"Dropped {len(dropped_empty)} basin(s) with no streamflow observations: " + f"{dropped_empty[:10]}{'...' if len(dropped_empty) > 10 else ''}" + ) + logger.info(f"Loaded {len(dfs)} basins with data from region={region}") + + if not dfs: + raise ValueError(f"No timeseries data loaded for region={region}") + + combined = pd.concat(dfs, ignore_index=True) + combined = combined.sort_values(["code", "date"]).reset_index(drop=True) + return combined + + +def load_hive_attributes( + base_path: str, + region: str, + basin_ids: list[str | int] | None = None, +) -> pd.DataFrame: + attr_path = _attributes_file(base_path, region) + if not attr_path.exists(): + raise FileNotFoundError(f"Attributes file not found: {attr_path}") + + df = pd.read_parquet(attr_path) + logger.info(f"Loaded hive attributes: {len(df)} rows, {len(df.columns)} cols") + + if "area" in df.columns: + df = df.rename(columns={"area": "area_km2"}) + if "gauge_lat" in df.columns: + df = df.rename(columns={"gauge_lat": "LAT", "gauge_lon": "LON"}) + + if "gauge_id" in df.columns: + region_prefix = f"{region}_" + code_strs = df["gauge_id"].apply( + lambda g: g[len(region_prefix) :] if isinstance(g, str) and g.startswith(region_prefix) else g + ) + try: + df["code"] = code_strs.astype(int) + except (ValueError, TypeError): + df["code"] = code_strs + df["CODE"] = df["code"] + + if basin_ids is not None: + if df["code"].dtype in ("int64", "int32", "float64"): + try: + basin_ids_typed = [int(b) for b in basin_ids] + except (ValueError, TypeError): + basin_ids_typed = list(basin_ids) + else: + basin_ids_typed = list(basin_ids) + df = df[df["code"].isin(basin_ids_typed)] + + df.index = df["CODE"] + numeric_cols = df.select_dtypes(include=["number"]).columns.tolist() + keep_cols = ["CODE"] + [c for c in numeric_cols if c != "CODE"] + return df[keep_cols] + + +def extract_discharge_from_timeseries(df: pd.DataFrame) -> pd.DataFrame: + cols = [c for c in ["date", "code", "discharge"] if c in df.columns] + return df[cols].copy() + + +def extract_forcing_from_timeseries( + df: pd.DataFrame, forcing_vars: list[str] | None = None +) -> pd.DataFrame: + base = ["date", "code"] + if forcing_vars is None: + exclude = {"date", "code", "discharge"} + forcing_vars = [c for c in df.columns if c not in exclude] + keep = base + [c for c in forcing_vars if c in df.columns] + return df[keep].copy() diff --git a/st_forecast/data_utils/loaders.py b/st_forecast/data_utils/loaders.py index 58b3223..b5de55c 100644 --- a/st_forecast/data_utils/loaders.py +++ b/st_forecast/data_utils/loaders.py @@ -8,7 +8,7 @@ import pandas as pd if TYPE_CHECKING: - from ..config.data_config import CaravanConfig + from ..config.data_config import CaravanConfig, HiveConfig from ..log_config import setup_logging from ..region_specific_utils import get_discharge_handler, get_static_handler @@ -1036,3 +1036,162 @@ def collect_all(self) -> dict: "static": static, "success": success, } + + +class HiveDataCollector: + """Collects data from a Hive-partitioned parquet dataset. + + Same interface as CaravanDataCollector but reads parquet from the layout + {base_path}/REGION_NAME={region}/data_type=.../... + """ + + def __init__( + self, + hive_config: "HiveConfig", + date_col: str = "date", + basin_id_col: str = "code", + static_id_col: str = "CODE", + ): + self.config = hive_config + self.date_col = date_col + self.basin_id_col = basin_id_col + self.static_id_col = static_id_col + self._timeseries_cache: pd.DataFrame | None = None + self._static_cache: pd.DataFrame | None = None + + def _load_timeseries(self) -> pd.DataFrame: + if self._timeseries_cache is not None: + return self._timeseries_cache + + from .hive_loaders import load_hive_timeseries + + self._timeseries_cache = load_hive_timeseries( + base_path=self.config.base_path, + region=self.config.region, + variable_mapping=self.config.variable_mapping, + n_basins=self.config.n_basins, + basin_ids=self.config.basin_ids, + start_date=self.config.start_date, + end_date=self.config.end_date, + drop_empty_discharge=self.config.drop_empty_discharge, + ) + return self._timeseries_cache + + def _load_static(self) -> pd.DataFrame: + if self._static_cache is not None: + return self._static_cache + + from .hive_loaders import load_hive_attributes + + ts = self._load_timeseries() + basin_ids = ts[self.basin_id_col].unique().tolist() + + self._static_cache = load_hive_attributes( + base_path=self.config.base_path, + region=self.config.region, + basin_ids=basin_ids, + ) + return self._static_cache + + def collect_discharge(self) -> pd.DataFrame: + from .hive_loaders import extract_discharge_from_timeseries + + ts = self._load_timeseries() + df = extract_discharge_from_timeseries(ts) + df[self.date_col] = pd.to_datetime(df[self.date_col]) + return df + + def collect_forcing(self) -> pd.DataFrame: + from .hive_loaders import extract_forcing_from_timeseries + + ts = self._load_timeseries() + forcing_vars = [ + v for v in self.config.variable_mapping.values() if v != "discharge" + ] + df = extract_forcing_from_timeseries(ts, forcing_vars) + df[self.date_col] = pd.to_datetime(df[self.date_col]) + return df + + def collect_static(self) -> pd.DataFrame: + return self._load_static() + + def collect_all(self) -> dict: + discharge = self.collect_discharge() + forcing = self.collect_forcing() + static = self.collect_static() + + success = { + "discharge": True, + "forcing": True, + "static": True, + "sca": False, + "swe": False, + "hs": False, + "rof": False, + "sla": False, + } + return { + "discharge": discharge, + "forcing": forcing, + "static": static, + "success": success, + } + + +def build_collector(run_config): + """Dispatch to the right collector based on run_config.data_source. + + Supports 'legacy' (RawDataCollector), 'caravan' (CaravanDataCollector), + and 'hive' (HiveDataCollector). + """ + data_source = getattr(run_config, "data_source", "legacy") + + if data_source == "caravan" and getattr(run_config, "caravan", None): + from ..config.data_config import CaravanConfig + + cfg = run_config.caravan + caravan_config = CaravanConfig( + base_path=cfg["base_path"], + region=cfg["region"], + n_basins=cfg.get("n_basins"), + basin_ids=cfg.get("basin_ids"), + start_date=cfg.get("start_date"), + end_date=cfg.get("end_date"), + ) + logger.info(f"Using Caravan data source: {caravan_config.region}") + return CaravanDataCollector(caravan_config) + + if data_source == "hive" and getattr(run_config, "hive", None): + from ..config.data_config import HiveConfig + + cfg = run_config.hive + default_mapping = HiveConfig.__dataclass_fields__[ + "variable_mapping" + ].default_factory() + hive_config = HiveConfig( + base_path=cfg["base_path"], + region=cfg["region"], + n_basins=cfg.get("n_basins"), + basin_ids=cfg.get("basin_ids"), + start_date=cfg.get("start_date"), + end_date=cfg.get("end_date"), + drop_empty_discharge=cfg.get("drop_empty_discharge", True), + variable_mapping=cfg.get("variable_mapping", default_mapping), + ) + logger.info(f"Using Hive data source: {hive_config.region}") + return HiveDataCollector(hive_config) + + paths_dict = { + "path_rivers": run_config.path_rivers, + "path_forcing": run_config.path_forcing, + "path_to_operational_forcing": run_config.path_to_operational_forcing, + "path_to_hindcast_forcing": run_config.path_to_hindcast_forcing, + "HRU_forcing": run_config.HRU_forcing, + "path_static": run_config.path_static, + "path_to_sla": run_config.path_to_sla, + "path_to_nir": run_config.path_to_nir, + "path_to_swe": run_config.path_to_swe, + "path_to_hs": run_config.path_to_hs, + "path_to_rof": run_config.path_to_rof, + } + return RawDataCollector(paths_dict, run_config.region) diff --git a/st_forecast/data_utils/preparation.py b/st_forecast/data_utils/preparation.py index 07a331a..6cad60d 100644 --- a/st_forecast/data_utils/preparation.py +++ b/st_forecast/data_utils/preparation.py @@ -40,6 +40,24 @@ def resolve_static_features( return [col for col in static_df.columns if col != static_id_col] +def filter_static_columns( + static_df: pd.DataFrame, + static_features_config: list[str], + static_id_col: str, +) -> pd.DataFrame: + """Restrict static dataframe to configured features (plus the ID column). + + When ``static_features_config`` is empty, the dataframe is returned unchanged + so the default "use everything" behaviour is preserved. + """ + if not static_features_config: + return static_df + resolved = resolve_static_features(static_df, static_features_config, static_id_col) + keep = [static_id_col] + [c for c in resolved if c != static_id_col] + keep = [c for c in keep if c in static_df.columns] + return static_df[keep] + + def prepare_train_data( temporal_df: pd.DataFrame, static_df: pd.DataFrame, @@ -237,8 +255,8 @@ def prepare_inference_data( ) # Step 8: Scale static features + static_features_config = config.get("static_features", []) if scale_static: - static_features_config = config.get("static_features", []) static_to_scale = resolve_static_features( static_df, static_features_config, static_id_col ) @@ -247,6 +265,9 @@ def prepare_inference_data( ) else: static_features = static_df + static_features = filter_static_columns( + static_features, static_features_config, static_id_col + ) # Step 9: Adjust feature names for shifted moving averages adjusted_features = adjust_feature_names(features, moving_windows, shift_moving_avg) @@ -495,8 +516,8 @@ def _build_timeseries_pipeline( ) # Step 8: Scale static features + static_features_config = config.get("static_features", []) if scale_static: - static_features_config = config.get("static_features", []) static_to_scale = resolve_static_features( static_df, static_features_config, static_id_col ) @@ -511,6 +532,9 @@ def _build_timeseries_pipeline( ) else: static_features = static_df + static_features = filter_static_columns( + static_features, static_features_config, static_id_col + ) # Adjust feature names for shifted moving averages adjusted_features = adjust_feature_names(features, moving_windows, shift_moving_avg) diff --git a/st_forecast/get_data.py b/st_forecast/get_data.py index b6d08aa..5317d73 100644 --- a/st_forecast/get_data.py +++ b/st_forecast/get_data.py @@ -309,6 +309,15 @@ def load_data(config): date_col=date_col, basin_id_col=basin_id_col, ) + elif config.data_source == "hive" and config.hive is not None: + from st_forecast.data_utils.loaders import HiveDataCollector + + logger.info(f"Using Hive data source: {config.hive.region}") + collector = HiveDataCollector( + config.hive, + date_col=date_col, + basin_id_col=basin_id_col, + ) else: path_config = config.to_legacy_dict() collector = RawDataCollector(path_config, config.region) diff --git a/st_forecast/pipeline/artifacts.py b/st_forecast/pipeline/artifacts.py index fd69e98..8198bc2 100644 --- a/st_forecast/pipeline/artifacts.py +++ b/st_forecast/pipeline/artifacts.py @@ -33,8 +33,9 @@ def normalize_model_type(model_type: str) -> str: class RunConfig: # Region/data paths region: str - data_source: str = "legacy" # "legacy" or "caravan" + data_source: str = "legacy" # "legacy", "caravan", or "hive" caravan: dict | None = None # Caravan dataset configuration + hive: dict | None = None # Hive-partitioned parquet dataset configuration path_forcing: str | None = None path_to_operational_forcing: str | None = None diff --git a/uv.lock b/uv.lock index ef07e72..4157b16 100644 --- a/uv.lock +++ b/uv.lock @@ -119,6 +119,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/4a/4c61d4c84cfd9befb6fa08a702535b27b21fff08c946bc2f6139decbf7f7/alembic-1.16.5-py3-none-any.whl", hash = "sha256:e845dfe090c5ffa7b92593ae6687c5cb1a101e91fa53868497dbd79847f9dbe3", size = 247355, upload-time = "2025-08-27T18:02:07.37Z" }, ] +[[package]] +name = "altair" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "narwhals" }, + { name = "packaging" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/1e/365a9144db3254f86f1b974660b9ede1e9a38c9dc0730e4a9b1192eec5d6/altair-6.1.0.tar.gz", hash = "sha256:dda699216cf85b040d968ae5a569ad45957616811e38760a85e5118269daca67", size = 765519, upload-time = "2026-04-21T13:08:46.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/63/5dacc8d8306c715088b897a479e551bc0779fd2f0f26c97fec5e36542b4e/altair-6.1.0-py3-none-any.whl", hash = "sha256:fdf5fd939512e5b2fc4441c82dfd2635e706defbd037db0ac429ef5ddce66c3b", size = 796996, upload-time = "2026-04-21T13:08:48.549Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -150,6 +166,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "bracex" version = "2.6" @@ -159,6 +184,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/2a/9186535ce58db529927f6cf5990a849aa9e052eea3e2cfefe20b9e1802da/bracex-2.6-py3-none-any.whl", hash = "sha256:0b0049264e7340b3ec782b5cb99beb325f36c3782a32e36e876452fd49a09952", size = 11508, upload-time = "2025-06-22T19:12:29.781Z" }, ] +[[package]] +name = "branca" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/14/9d409124bda3f4ab7af3802aba07181d1fd56aa96cc4b999faea6a27a0d2/branca-0.8.2.tar.gz", hash = "sha256:e5040f4c286e973658c27de9225c1a5a7356dd0702a7c8d84c0f0dfbde388fe7", size = 27890, upload-time = "2025-10-06T10:28:20.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/50/fc9680058e63161f2f63165b84c957a0df1415431104c408e8104a3a18ef/branca-0.8.2-py3-none-any.whl", hash = "sha256:2ebaef3983e3312733c1ae2b793b0a8ba3e1c4edeb7598e10328505280cf2f7c", size = 26193, upload-time = "2025-10-06T10:28:19.255Z" }, +] + [[package]] name = "bump-my-version" version = "1.2.7" @@ -179,6 +216,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/ed/ad1755f82cd5a0baafe342e7154696a93e57f04f86515402f14e5beceb36/bump_my_version-1.2.7-py3-none-any.whl", hash = "sha256:16f89360f979c0a8eb3249ebe3e13ae4f0cb5481d7bb58e12a9f66996922acfd", size = 60013, upload-time = "2026-02-14T13:44:58.318Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8b/0d3945a13955303b81272f759a0331e54c5c793da455e6f5706b89d2639c/cachetools-7.1.4.tar.gz", hash = "sha256:437f55a4e0c1b01a4f3077cc470e6991d47430970e36fbcb77e2be0df4fc1cd6", size = 40085, upload-time = "2026-05-21T22:40:43.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/7b/1fc1c09cc0756cf25861a3be10565915953876da48bb228fb9a672b20a42/cachetools-7.1.4-py3-none-any.whl", hash = "sha256:323dc4127934744db5b54eb4924482d7edafbf9554e820d1531c2e08c0e4ef54", size = 16761, upload-time = "2026-05-21T22:40:41.845Z" }, +] + [[package]] name = "cartopy" version = "0.25.0" @@ -482,6 +528,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, ] +[[package]] +name = "folium" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "branca" }, + { name = "jinja2" }, + { name = "numpy" }, + { name = "requests" }, + { name = "xyzservices" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/76/84a1b1b00ce71f9c0c44af7d80f310c02e2e583591fe7d4cb03baecd0d3f/folium-0.20.0.tar.gz", hash = "sha256:a0d78b9d5a36ba7589ca9aedbd433e84e9fcab79cd6ac213adbcff922e454cb9", size = 109932, upload-time = "2025-06-16T20:22:51.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/a8/5f764f333204db0390362a4356d03a43626997f26818a0e9396f1b3bd8c9/folium-0.20.0-py2.py3-none-any.whl", hash = "sha256:f0bc2a92acde20bca56367aa5c1c376c433f450608d058daebab2fc9bf8198bf", size = 113394, upload-time = "2025-06-16T20:22:50.318Z" }, +] + [[package]] name = "fonttools" version = "4.60.1" @@ -660,6 +722,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/15/cf2a69ade4b194aa524ac75112d5caac37414b20a3a03e6865dfe0bd1539/geopy-2.4.1-py3-none-any.whl", hash = "sha256:ae8b4bc5c1131820f4d75fce9d4aaaca0c85189b3aa5d64c3dcaf5e3b7b882a7", size = 125437, upload-time = "2023-11-23T21:49:30.421Z" }, ] +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.50" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, +] + [[package]] name = "greenlet" version = "3.2.4" @@ -744,6 +830,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httptools" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/d2/c3eedaef57de65c3cc5f8dc244cf12d09c84ad258a479055aad6db23206c/httptools-0.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ed377e64805bdba4943c82717333f8f8603a13b09aff9cead2717c6c817fb168", size = 208428, upload-time = "2026-05-25T22:16:59.717Z" }, + { url = "https://files.pythonhosted.org/packages/f1/94/dfe435d90d0ef61ec0f2cc3d480eef78c59727c6c2ce039f433882f6131a/httptools-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9518c406d7b310f05adb1a37f80acabac40504a575d7c0da6d3e365c695ac20d", size = 113366, upload-time = "2026-05-25T22:17:00.795Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d4/13025f1a56e615dcb331e0bbe2d9a1143212b58c263385fc5d2e558f5bac/httptools-0.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376", size = 464676, upload-time = "2026-05-25T22:17:02.014Z" }, + { url = "https://files.pythonhosted.org/packages/bf/95/4c1c26c0b985f8a3331682d802598f14e32dc41bf7509266eb2c04ad4801/httptools-0.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d", size = 464235, upload-time = "2026-05-25T22:17:03.109Z" }, + { url = "https://files.pythonhosted.org/packages/a2/82/6735be2b0ca527718c431cdb8e5f70c3862c0844a687df0f572c51e11497/httptools-0.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:52dd695b865fe96d9d2b16b64a895f3f57bf3cb064e8383cd3b5713a069e8085", size = 449809, upload-time = "2026-05-25T22:17:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f9/5811c74f37a758c8a4aa3dc430375119d335947e883efc4664d8f3559a41/httptools-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:20b4aac66ff65f7db06a375808b78f42a94970aa22e826b3cb2b43eb09174124", size = 452174, upload-time = "2026-05-25T22:17:05.476Z" }, + { url = "https://files.pythonhosted.org/packages/cc/94/97b75870dea07b71e3ec535cebe525b08d723152e4c7d13fa887e51f4de2/httptools-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1b4c8e7a489a0d750d91894e9a8cdc295838f1924c0ca903ae993456fddec07", size = 90991, upload-time = "2026-05-25T22:17:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/14/88/1d21a36da8f5cb0fa49eafd4b169eba5608d57e75bbcf61845cbc6243216/httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d", size = 208247, upload-time = "2026-05-25T22:17:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5", size = 113064, upload-time = "2026-05-25T22:17:09.136Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2", size = 523851, upload-time = "2026-05-25T22:17:10.106Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e4/f90a0df0b83beff265b7e3b65f2a4cefd95792d4be0ac3e16049f2acd3c2/httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09", size = 518842, upload-time = "2026-05-25T22:17:11.218Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2d/0c9ac76dd2c893841fbf6498d6acec4f2442e1b7067f6e3e316a80e494e8/httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a", size = 501238, upload-time = "2026-05-25T22:17:12.728Z" }, + { url = "https://files.pythonhosted.org/packages/ca/42/906adc91ae3a5fa9c59c0a2f21c139725bd7e5b41ae6acd485cd14123ebf/httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745", size = 509567, upload-time = "2026-05-25T22:17:13.842Z" }, + { url = "https://files.pythonhosted.org/packages/05/0b/4240efeb672751ee5b9b380cb0e3fdc050bc05f68adc7a8aefc4fcd9a69a/httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150", size = 90918, upload-time = "2026-05-25T22:17:15.155Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e5/8cfcabc5546e8022f168be28bcdaa128a240a0befdd03b59d558b4f18bd6/httptools-0.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8", size = 205148, upload-time = "2026-05-25T22:17:16.333Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0e/0fb14848c19a686c8062ff9067c1a48793e3224b47bc5b201535b6036fce/httptools-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c", size = 111368, upload-time = "2026-05-25T22:17:17.586Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/46f1cecf06b9bbde8e4b8c88034ac7908989e5ff7a3a388ef38392949c1f/httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7", size = 486447, upload-time = "2026-05-25T22:17:18.564Z" }, + { url = "https://files.pythonhosted.org/packages/77/00/258bfc0837221f81d9725c45f9b948a6a6b2994a147a4fb66e85100c668f/httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d", size = 482448, upload-time = "2026-05-25T22:17:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/04/ab/d1cef3b5523f4d272a70f42a776c3169a2dddfe3a54de4b2ce4a36341528/httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681", size = 464460, upload-time = "2026-05-25T22:17:20.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5d1d072442277bb2b3434e0e60690b8e8c23840ef7de8b6ea54040a536d3/httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683", size = 471312, upload-time = "2026-05-25T22:17:22.085Z" }, + { url = "https://files.pythonhosted.org/packages/0d/66/b96623b27e51a68199ef4efdda0613cced9233fe3062ac74e50749c5ad37/httptools-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1", size = 90117, upload-time = "2026-05-25T22:17:23.074Z" }, + { url = "https://files.pythonhosted.org/packages/1a/12/fa3fbf5f9517b273edea2dc982aa82a8c634091e67c590792b729017bc6f/httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6", size = 206183, upload-time = "2026-05-25T22:17:24.004Z" }, + { url = "https://files.pythonhosted.org/packages/30/fc/5e7c4cb443370f2090a3aba0453a07384d29ff66b7435bb90e77e1037599/httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b", size = 112079, upload-time = "2026-05-25T22:17:25.216Z" }, + { url = "https://files.pythonhosted.org/packages/ba/53/771bd891eb0f236f32145d6a1775777ec85745f3cc983a1f23d1a3b8ddfe/httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0", size = 481596, upload-time = "2026-05-25T22:17:26.186Z" }, + { url = "https://files.pythonhosted.org/packages/62/42/94e15bc68ce3d423243c45d7f1b0c7561f13844f97dc52ae23182fb65628/httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e", size = 480865, upload-time = "2026-05-25T22:17:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7c/fe2980fc03723272e30f135b62360b075f513dfe7cc73aef36c7f04012bd/httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b", size = 463189, upload-time = "2026-05-25T22:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/47fc5fff68acd1bfa20b4734059c9a06cadb88119dcd5258b5b0d21d91c8/httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0", size = 466610, upload-time = "2026-05-25T22:17:29.816Z" }, + { url = "https://files.pythonhosted.org/packages/60/bd/07b13c93ffd9bec9546e0d43f8e19378dd696dbd278511406bc07371ef1f/httptools-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527", size = 92705, upload-time = "2026-05-25T22:17:31.133Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c4/121648f68ce066d7bd762d6b6d97e620847642d38d54f3d90ff11d947629/httptools-0.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568", size = 215023, upload-time = "2026-05-25T22:17:32.401Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b0/312a062ae741ae3e8baa8c8bf20be81b2e67337b259ab4349bebc7b6142e/httptools-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b", size = 117405, upload-time = "2026-05-25T22:17:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/fc/37/fccd705f795386bb05bf413012fecff2a33e5aa8c2f069096de3e9fd8702/httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca", size = 558497, upload-time = "2026-05-25T22:17:34.732Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/f172e8003576de35f5ba77ff417cf0e34429d35dc014deef15afa337a72c/httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f", size = 571585, upload-time = "2026-05-25T22:17:35.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b9/f5564760af99f3dbbf3f9104dc00e5da27e96cf433c6bdcf77617f70bf3f/httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d", size = 543297, upload-time = "2026-05-25T22:17:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/99/67/8d9f2c313618e161b82f3873188e7196126da1d6e29688df40eb3997c77a/httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081", size = 539535, upload-time = "2026-05-25T22:17:38.032Z" }, + { url = "https://files.pythonhosted.org/packages/48/63/b906c01e53f50d432c0defe43ce52764a111dc1bdd028bafbeb54dcfd008/httptools-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77", size = 108209, upload-time = "2026-05-25T22:17:39.473Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -777,6 +906,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -798,6 +936,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/e8/685f47e0d754320684db4425a0967f7d3fa70126bffd76110b7009a0090f/joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241", size = 308396, upload-time = "2025-08-27T12:15:45.188Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "kiwisolver" version = "1.4.9" @@ -1682,6 +1847,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/e7/ae39f538fd6844e982063c3a5e4598b8ced43b9633baa3a85ef33af8c05c/pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8", size = 6984598, upload-time = "2025-07-01T09:16:27.732Z" }, ] +[[package]] +name = "plotly" +version = "6.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/7f/0f100df1172aadf88a929a9dbb902656b0880ba4b960fe5224867159d8f4/plotly-6.7.0.tar.gz", hash = "sha256:45eea0ff27e2a23ccd62776f77eb43aa1ca03df4192b76036e380bb479b892c6", size = 6911286, upload-time = "2026-04-09T20:36:45.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/ad/cba91b3bcf04073e4d1655a5c1710ef3f457f56f7d1b79dcc3d72f4dd912/plotly-6.7.0-py3-none-any.whl", hash = "sha256:ac8aca1c25c663a59b5b9140a549264a5badde2e057d79b8c772ae2920e32ff0", size = 9898444, upload-time = "2026-04-09T20:36:39.812Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1981,6 +2159,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/4b/ccc026168948fec4f7555b9164c724cf4125eac006e176541483d2c959be/pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237", size = 58929, upload-time = "2026-02-19T13:45:06.034Z" }, ] +[[package]] +name = "pydeck" +version = "0.9.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/df/4e9e7f20f8034a37c6571c93809f6d22388c39978c98d174d656c1a18fd2/pydeck-0.9.2.tar.gz", hash = "sha256:c10d9035e81ead6385264cac8d19402471f6866a15ca1f7df1400f52142bcf87", size = 5849672, upload-time = "2026-04-16T18:30:30.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/24/b30ee7d723100fd822de1bb4c0adea62f3419884a75a536f35f355d1e7c0/pydeck-0.9.2-py2.py3-none-any.whl", hash = "sha256:8213dfeacc5f6bfe6825f61c8ee34e3850e8a31fc43924379ec98edb34a75b25", size = 11305615, upload-time = "2026-04-16T18:30:28.133Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -2176,6 +2367,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/fe/70bd71a6738b09a0bdf6480ca6436b167469ca4578b2a0efbe390b4b0e70/python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904", size = 45678, upload-time = "2026-05-17T17:29:47.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/cb/769cfc37177252872a45a71f3fbdde9d51b471a3f3c14bfe95dde3407386/python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69", size = 29640, upload-time = "2026-05-17T17:29:45.69Z" }, +] + [[package]] name = "pytorch-lightning" version = "2.5.5" @@ -2378,6 +2578,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d1/8b017856e63ccaff3cbd0e82490dbb01363a42f3a462a41b1d8a391e1443/rasterio-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f4b9c2c3b5f10469eb9588f105086e68f0279e62cc9095c4edd245e3f9b88c8a", size = 29418321, upload-time = "2026-01-05T16:06:44.758Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "requests" version = "2.32.5" @@ -2420,6 +2634,114 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/e5/d708d262b600a352abe01c2ae360d8ff75b0af819b78e9af293191d928e6/rich_click-1.9.7-py3-none-any.whl", hash = "sha256:2f99120fca78f536e07b114d3b60333bc4bb2a0969053b1250869bcdc1b5351b", size = 71491, upload-time = "2026-01-31T04:29:26.777Z" }, ] +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, + { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, + { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, + { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, + { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, + { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, + { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, + { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, + { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, + { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, + { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, + { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, + { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, +] + [[package]] name = "ruff" version = "0.13.2" @@ -2694,6 +3016,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/81/9ef641ff4e12cbcca30e54e72fb0951a2ba195d0cda0ba4100e532d929db/slicer-0.0.8-py3-none-any.whl", hash = "sha256:6c206258543aecd010d497dc2eca9d2805860a0b3758673903456b7df7934dc3", size = 15251, upload-time = "2024-03-09T07:03:07.708Z" }, ] +[[package]] +name = "smmap" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.43" @@ -2759,10 +3090,14 @@ dev = [ { name = "bump-my-version" }, { name = "cartopy" }, { name = "contextily" }, + { name = "folium" }, { name = "geopandas" }, { name = "matplotlib-scalebar" }, + { name = "plotly" }, { name = "pytest" }, { name = "ruff" }, + { name = "streamlit" }, + { name = "streamlit-folium" }, ] [package.metadata] @@ -2790,10 +3125,27 @@ dev = [ { name = "bump-my-version", specifier = ">=1.2.7" }, { name = "cartopy", specifier = ">=0.25.0" }, { name = "contextily", specifier = ">=1.7.0" }, + { name = "folium", specifier = ">=0.20.0" }, { name = "geopandas", specifier = ">=1.1.3" }, { name = "matplotlib-scalebar", specifier = ">=0.9.0" }, + { name = "plotly", specifier = ">=6.7.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "ruff", specifier = ">=0.13.2" }, + { name = "streamlit", specifier = ">=1.57.0" }, + { name = "streamlit-folium", specifier = ">=0.27.2" }, +] + +[[package]] +name = "starlette" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/66/4d20cdf39a8d6a51e663b7038e3b828ff211d3891a43a713fe7e4643f3a8/starlette-1.1.0.tar.gz", hash = "sha256:e83c7fe0ddecd8719c5b840080325aec0260acec86e9832899e377b91d65e90f", size = 2660060, upload-time = "2026-05-23T16:55:41.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/79/920b8e0a8b20f793e8d64855095cb8febabf6175b8550b6f7a547d813891/starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382", size = 72899, upload-time = "2026-05-23T16:55:39.201Z" }, ] [[package]] @@ -2835,6 +3187,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/c0/b28d0fd0347ea38d3610052f479e4b922eb33bb8790817f93cd89e6e08ba/statsmodels-0.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:95af7a9c4689d514f4341478b891f867766f3da297f514b8c4adf08f4fa61d03", size = 9648961, upload-time = "2025-10-30T13:47:24.303Z" }, ] +[[package]] +name = "streamlit" +version = "1.57.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altair" }, + { name = "anyio" }, + { name = "blinker" }, + { name = "cachetools" }, + { name = "click" }, + { name = "gitpython" }, + { name = "httptools" }, + { name = "itsdangerous" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "pyarrow" }, + { name = "pydeck" }, + { name = "python-multipart" }, + { name = "requests" }, + { name = "starlette" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "typing-extensions" }, + { name = "uvicorn" }, + { name = "watchdog", marker = "sys_platform != 'darwin'" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/f8/b2daf7a5f8ae15527daf94406e771bb6075e958a01c3dde9eba79dc3c9a3/streamlit-1.57.0.tar.gz", hash = "sha256:0b028d305c1a1a757071b2c9504966787602842fc8af6e873795ca58d2b4d12f", size = 8678859, upload-time = "2026-04-28T22:13:32.238Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/1a/3ca2293d8552bacea3e67e9600d2d1df7df4a325059769ad83d91c279595/streamlit-1.57.0-py3-none-any.whl", hash = "sha256:0d1d41972aeade5637dbb0e7f0eefa5312272f85304923d240a1b1f0475249c8", size = 9194216, upload-time = "2026-04-28T22:13:29.624Z" }, +] + +[[package]] +name = "streamlit-folium" +version = "0.27.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "branca" }, + { name = "folium" }, + { name = "jinja2" }, + { name = "streamlit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/d8/f19603163670473f1d937fe065cef83c34ea581dd5cb50a908754896b9f5/streamlit_folium-0.27.2.tar.gz", hash = "sha256:d3ab790732fa7adb7cda68ec3170116ab6f276998fd46ac79686a4d88c0f4c84", size = 528270, upload-time = "2026-04-29T15:58:58.637Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/5a/46c029aef79656fd9786fc1d6c2572855242da378d7f57fa525f75da0441/streamlit_folium-0.27.2-py3-none-any.whl", hash = "sha256:2c6a4168dcbbeadf4c9521d4fecef0984e542081415f91dde6b05b536a5f2b27", size = 530501, upload-time = "2026-04-29T15:58:59.781Z" }, +] + [[package]] name = "suntime" version = "1.3.2" @@ -2859,6 +3261,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "tensorboardx" version = "2.6.4" @@ -2882,6 +3293,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomlkit" version = "0.14.0" @@ -3018,6 +3438,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, ] +[[package]] +name = "uvicorn" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/bf/f6544ba992ddb9a6077343a576f9844f7f8f06ab819aefd00206e9255f18/uvicorn-0.48.0.tar.gz", hash = "sha256:a5504207195d08c2511bf9125ede5ac4a4b71725d519e758d01dcf0bc2d31c37", size = 91074, upload-time = "2026-05-24T12:08:41.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/be/72532be3da7acc5fdfbccdb95215cd04f995a0886532a5b423f929cda4cc/uvicorn-0.48.0-py3-none-any.whl", hash = "sha256:48097851328b87ec36117d3d575234519eb58c2b22d79666e9bbc6c49a761dad", size = 71410, upload-time = "2026-05-24T12:08:40.258Z" }, +] + +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + [[package]] name = "wcmatch" version = "10.1" @@ -3039,6 +3490,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, ] +[[package]] +name = "websockets" +version = "16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/db/de907251b4ff46ae804ad0409809504153b3f30984daf82a1d84a9875830/websockets-16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8", size = 177340, upload-time = "2026-01-10T09:22:34.539Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fa/abe89019d8d8815c8781e90d697dec52523fb8ebe308bf11664e8de1877e/websockets-16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad", size = 175022, upload-time = "2026-01-10T09:22:36.332Z" }, + { url = "https://files.pythonhosted.org/packages/58/5d/88ea17ed1ded2079358b40d31d48abe90a73c9e5819dbcde1606e991e2ad/websockets-16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d", size = 175319, upload-time = "2026-01-10T09:22:37.602Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ae/0ee92b33087a33632f37a635e11e1d99d429d3d323329675a6022312aac2/websockets-16.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe", size = 184631, upload-time = "2026-01-10T09:22:38.789Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c5/27178df583b6c5b31b29f526ba2da5e2f864ecc79c99dae630a85d68c304/websockets-16.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b", size = 185870, upload-time = "2026-01-10T09:22:39.893Z" }, + { url = "https://files.pythonhosted.org/packages/87/05/536652aa84ddc1c018dbb7e2c4cbcd0db884580bf8e95aece7593fde526f/websockets-16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5", size = 185361, upload-time = "2026-01-10T09:22:41.016Z" }, + { url = "https://files.pythonhosted.org/packages/6d/e2/d5332c90da12b1e01f06fb1b85c50cfc489783076547415bf9f0a659ec19/websockets-16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64", size = 184615, upload-time = "2026-01-10T09:22:42.442Z" }, + { url = "https://files.pythonhosted.org/packages/77/fb/d3f9576691cae9253b51555f841bc6600bf0a983a461c79500ace5a5b364/websockets-16.0-cp311-cp311-win32.whl", hash = "sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6", size = 178246, upload-time = "2026-01-10T09:22:43.654Z" }, + { url = "https://files.pythonhosted.org/packages/54/67/eaff76b3dbaf18dcddabc3b8c1dba50b483761cccff67793897945b37408/websockets-16.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac", size = 178684, upload-time = "2026-01-10T09:22:44.941Z" }, + { url = "https://files.pythonhosted.org/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00", size = 177365, upload-time = "2026-01-10T09:22:46.787Z" }, + { url = "https://files.pythonhosted.org/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79", size = 175038, upload-time = "2026-01-10T09:22:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39", size = 175328, upload-time = "2026-01-10T09:22:49.809Z" }, + { url = "https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c", size = 184915, upload-time = "2026-01-10T09:22:51.071Z" }, + { url = "https://files.pythonhosted.org/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f", size = 186152, upload-time = "2026-01-10T09:22:52.224Z" }, + { url = "https://files.pythonhosted.org/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1", size = 185583, upload-time = "2026-01-10T09:22:53.443Z" }, + { url = "https://files.pythonhosted.org/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2", size = 184880, upload-time = "2026-01-10T09:22:55.033Z" }, + { url = "https://files.pythonhosted.org/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89", size = 178261, upload-time = "2026-01-10T09:22:56.251Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea", size = 178693, upload-time = "2026-01-10T09:22:57.478Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9", size = 177364, upload-time = "2026-01-10T09:22:59.333Z" }, + { url = "https://files.pythonhosted.org/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230", size = 175039, upload-time = "2026-01-10T09:23:01.171Z" }, + { url = "https://files.pythonhosted.org/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c", size = 175323, upload-time = "2026-01-10T09:23:02.341Z" }, + { url = "https://files.pythonhosted.org/packages/bd/28/0a25ee5342eb5d5f297d992a77e56892ecb65e7854c7898fb7d35e9b33bd/websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5", size = 184975, upload-time = "2026-01-10T09:23:03.756Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/27ea52741752f5107c2e41fda05e8395a682a1e11c4e592a809a90c6a506/websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82", size = 186203, upload-time = "2026-01-10T09:23:05.01Z" }, + { url = "https://files.pythonhosted.org/packages/37/e5/8e32857371406a757816a2b471939d51c463509be73fa538216ea52b792a/websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8", size = 185653, upload-time = "2026-01-10T09:23:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/9b/67/f926bac29882894669368dc73f4da900fcdf47955d0a0185d60103df5737/websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f", size = 184920, upload-time = "2026-01-10T09:23:07.492Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a1/3d6ccdcd125b0a42a311bcd15a7f705d688f73b2a22d8cf1c0875d35d34a/websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a", size = 178255, upload-time = "2026-01-10T09:23:09.245Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ae/90366304d7c2ce80f9b826096a9e9048b4bb760e44d3b873bb272cba696b/websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156", size = 178689, upload-time = "2026-01-10T09:23:10.483Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406, upload-time = "2026-01-10T09:23:12.178Z" }, + { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085, upload-time = "2026-01-10T09:23:13.511Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328, upload-time = "2026-01-10T09:23:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044, upload-time = "2026-01-10T09:23:15.939Z" }, + { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279, upload-time = "2026-01-10T09:23:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711, upload-time = "2026-01-10T09:23:18.372Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982, upload-time = "2026-01-10T09:23:19.652Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915, upload-time = "2026-01-10T09:23:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381, upload-time = "2026-01-10T09:23:22.715Z" }, + { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737, upload-time = "2026-01-10T09:23:24.523Z" }, + { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268, upload-time = "2026-01-10T09:23:25.781Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486, upload-time = "2026-01-10T09:23:27.033Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331, upload-time = "2026-01-10T09:23:28.259Z" }, + { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501, upload-time = "2026-01-10T09:23:29.449Z" }, + { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062, upload-time = "2026-01-10T09:23:31.368Z" }, + { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356, upload-time = "2026-01-10T09:23:32.627Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085, upload-time = "2026-01-10T09:23:33.816Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531, upload-time = "2026-01-10T09:23:35.016Z" }, + { url = "https://files.pythonhosted.org/packages/72/07/c98a68571dcf256e74f1f816b8cc5eae6eb2d3d5cfa44d37f801619d9166/websockets-16.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d", size = 174947, upload-time = "2026-01-10T09:23:36.166Z" }, + { url = "https://files.pythonhosted.org/packages/7e/52/93e166a81e0305b33fe416338be92ae863563fe7bce446b0f687b9df5aea/websockets-16.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03", size = 175260, upload-time = "2026-01-10T09:23:37.409Z" }, + { url = "https://files.pythonhosted.org/packages/56/0c/2dbf513bafd24889d33de2ff0368190a0e69f37bcfa19009ef819fe4d507/websockets-16.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da", size = 176071, upload-time = "2026-01-10T09:23:39.158Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8f/aea9c71cc92bf9b6cc0f7f70df8f0b420636b6c96ef4feee1e16f80f75dd/websockets-16.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c", size = 176968, upload-time = "2026-01-10T09:23:41.031Z" }, + { url = "https://files.pythonhosted.org/packages/9a/3f/f70e03f40ffc9a30d817eef7da1be72ee4956ba8d7255c399a01b135902a/websockets-16.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767", size = 178735, upload-time = "2026-01-10T09:23:42.259Z" }, + { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, +] + [[package]] name = "xarray" version = "2025.9.1" From f7a888b378edd6b866f6b516acd746b8def3a74d Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Thu, 4 Jun 2026 09:30:02 +0200 Subject: [PATCH 11/20] matmul precision --- st_forecast/model_helper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/st_forecast/model_helper.py b/st_forecast/model_helper.py index 3f61c12..2601d18 100644 --- a/st_forecast/model_helper.py +++ b/st_forecast/model_helper.py @@ -26,6 +26,9 @@ torch.serialization.add_safe_globals([QuantileRegression]) +if torch.cuda.is_available(): + torch.set_float32_matmul_precision("high") + _ENCODER_MAP: dict[str, dict | None] = { "week": {"cyclic": {"future": ["week"]}}, "month": {"cyclic": {"future": ["month"]}}, From 6e2798f0c6a1f8e3ed4cb114817229dc0d6bca34 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:43:30 +0200 Subject: [PATCH 12/20] kaz integration --- scripts/explore_kaz.py | 428 ++++++++++++++++-- st_forecast/evaluation/__init__.py | 10 + st_forecast/evaluation/evaluator.py | 72 ++- st_forecast/evaluation/metric_functions.py | 64 +++ st_forecast/evaluation/performance_metrics.py | 21 +- st_forecast/visualization/compare_runs.py | 113 +++++ st_forecast/visualization/metrics.py | 1 + tests/test_evaluation/test_evaluator.py | 107 ++++- .../test_performance_metrics.py | 100 ++++ 9 files changed, 872 insertions(+), 44 deletions(-) diff --git a/scripts/explore_kaz.py b/scripts/explore_kaz.py index 5b0fab5..b746f2e 100644 --- a/scripts/explore_kaz.py +++ b/scripts/explore_kaz.py @@ -3,6 +3,7 @@ Launch with: uv run streamlit run scripts/explore_kaz.py + uv run streamlit run scripts/explore_kaz.py -- --pred-dir ./runs/kaz/lstm Layout: - Sidebar: dataset path, region, map-colour selector, aridity filter. @@ -11,10 +12,12 @@ hypotheses), aridity, and streamflow histogram. - Tab "Cross-basin": the area / aridity / specific-runoff / RR-histogram plots for all gauged basins. +- Tab "Model performance": NSE map + distribution for the configured run. """ from __future__ import annotations +import argparse import logging from pathlib import Path @@ -42,6 +45,10 @@ "KAZ_DATA/sandro-basins-kaz" ) DEFAULT_REGION = "kaz" +DEFAULT_PRED_DIR = "runs/kaz/lstm" +TEST_START = pd.Timestamp("2016-01-02") +TEST_END = pd.Timestamp("2020-12-30") +QUANTILE_COLS = ["Q5", "Q10", "Q25", "Q50", "Q75", "Q90", "Q95"] VAR_MAP = { "streamflow": "discharge", "total_precipitation_sum": "P", @@ -52,6 +59,30 @@ ZERO_FLOW_THRESHOLD = 0.001 # m³/s — anything below treated as "essentially dry" +def _parse_cli_args() -> argparse.Namespace: + """Parse CLI args passed after `streamlit run script.py --`. + + `parse_known_args` ignores any extra flags streamlit forwards. + """ + parser = argparse.ArgumentParser( + prog="explore_kaz", + description="Interactive KAZ basin / LSTM forecast dashboard.", + ) + parser.add_argument( + "--pred-dir", + default=DEFAULT_PRED_DIR, + help=( + "Path to an LSTM run directory (must contain predictions/, " + "metrics/, scalers/). Default: %(default)s." + ), + ) + args, _ = parser.parse_known_args() + return args + + +CLI_ARGS = _parse_cli_args() + + # --------------------------------------------------------------------------- # Data loading (all cached) # --------------------------------------------------------------------------- @@ -143,6 +174,69 @@ def nearest_gauge(inv: pd.DataFrame, lat: float, lng: float) -> str: return str(inv.loc[d2.idxmin(), "gauge_id"]) +def _handle_map_click( + map_result: dict | None, inv: pd.DataFrame, click_key: str +) -> None: + """Update selected_gauge from a fresh map click, keeping the selectbox in sync. + + `st_folium` keeps returning the same `last_object_clicked` across reruns; we + store it in session state under `click_key` and only react when the click + coordinates actually change. The widget state for the selectbox must be + written *before* `st.rerun()` or the next render reverts the selection. + """ + click = (map_result or {}).get("last_object_clicked") + if not click: + return + if click == st.session_state.get(click_key): + return + st.session_state[click_key] = click + clicked = nearest_gauge(inv, click["lat"], click["lng"]) + if clicked and clicked != st.session_state.get("selected_gauge"): + st.session_state.selected_gauge = clicked + st.session_state.gauge_selectbox = clicked + st.rerun() + + +# --------------------------------------------------------------------------- +# LSTM forecast artifacts +# --------------------------------------------------------------------------- + + +def _code_from_gauge_id(gauge_id: str) -> int: + return int(gauge_id.removeprefix("kaz_")) + + +@st.cache_data(show_spinner="Loading LSTM metrics…") +def load_test_metrics(run_dir: str) -> pd.DataFrame: + df = pd.read_csv(Path(run_dir) / "metrics" / "test_metrics.csv") + df["code"] = df["code"].astype(int) + df["gauge_id"] = "kaz_" + df["code"].astype(str) + return df + + +@st.cache_data(show_spinner="Loading LSTM predictions…") +def load_predictions(run_dir: str) -> pd.DataFrame: + df = pd.read_parquet(Path(run_dir) / "predictions" / "test_predictions.parquet") + df["code"] = df["code"].astype(int) + df["date"] = pd.to_datetime(df["date"]) + return df + + +def predictions_for_basin(preds: pd.DataFrame, code: int, lead: int) -> pd.DataFrame: + """Return quantile predictions in mm/day. + + Values in `test_predictions.parquet` are already in the original target + unit (mm/day specific runoff) — the run's `discharge_scalers.csv` is + metadata only and must NOT be applied here. Sanity-checked: raw Q50 + matches the observed streamflow series with NSE > 0.96 (consistent with + the reported NSE in `metrics/test_metrics.csv`). + """ + sub = preds[(preds["code"] == code) & (preds["forecast_step"] == lead)] + if sub.empty: + return pd.DataFrame() + return sub[["date", *QUANTILE_COLS]].set_index("date").sort_index() + + # --------------------------------------------------------------------------- # Map # --------------------------------------------------------------------------- @@ -186,7 +280,7 @@ def build_map( highlight_function=lambda _: {"weight": 1.2, "color": "#222"}, ).add_to(m) - # Overlay the selected basin's polygon in a bold contrasting colour + # Overlay the selected basin's polygon with a bold orange outline if selected_gauge_id is not None: sel_shape = shapes[shapes["gauge_id"] == selected_gauge_id] if not sel_shape.empty: @@ -194,10 +288,10 @@ def build_map( sel_shape.to_json(), name="Selected basin", style_function=lambda _: { - "fillColor": "#e74c3c", - "fillOpacity": 0.20, - "color": "#c0392b", - "weight": 4.0, + "fillColor": "#ff8c00", + "fillOpacity": 0.18, + "color": "#e67e22", + "weight": 5.0, "opacity": 1.0, }, ).add_to(m) @@ -273,12 +367,109 @@ def build_map( return m +def build_nse_map( + inv: pd.DataFrame, + shapes: gpd.GeoDataFrame, + metrics: pd.DataFrame, + lead: int, + selected_gauge_id: str | None, +) -> folium.Map: + centre_lat = float(inv["gauge_lat"].mean()) + centre_lon = float(inv["gauge_lon"].mean()) + m = folium.Map( + location=[centre_lat, centre_lon], + zoom_start=5, + tiles="cartodbpositron", + control_scale=True, + ) + + nse_by_gauge = ( + metrics[metrics["forecast_step"] == lead].set_index("gauge_id")["NSE"].to_dict() + ) + cmap = bcm.linear.RdYlGn_09.scale(-0.5, 1.0) + cmap.caption = f"NSE (lead {lead}d)" + + def style_fn(feat: dict) -> dict: + gid = feat["properties"].get("gauge_id") + nse = nse_by_gauge.get(gid) + if nse is None or pd.isna(nse): + return { + "fillColor": "#dddddd", + "fillOpacity": 0.10, + "color": "#999999", + "weight": 0.4, + } + clamped = max(-0.5, min(1.0, float(nse))) + return { + "fillColor": cmap(clamped), + "fillOpacity": 0.70, + "color": "#333333", + "weight": 0.7, + } + + shapes_with_nse = shapes.copy() + shapes_with_nse["NSE"] = shapes_with_nse["gauge_id"].map(nse_by_gauge) + folium.GeoJson( + shapes_with_nse.to_json(), + name="NSE by basin", + style_function=style_fn, + highlight_function=lambda _: {"weight": 2.0, "color": "#111"}, + tooltip=folium.GeoJsonTooltip( + fields=["gauge_id", "NSE"], + aliases=["Basin", f"NSE (lead {lead}d)"], + localize=True, + sticky=True, + ), + ).add_to(m) + + if selected_gauge_id is not None: + sel_shape = shapes[shapes["gauge_id"] == selected_gauge_id] + if not sel_shape.empty: + folium.GeoJson( + sel_shape.to_json(), + name="Selected basin", + style_function=lambda _: { + "fillColor": "#ff8c00", + "fillOpacity": 0.10, + "color": "#e67e22", + "weight": 5.0, + "opacity": 1.0, + }, + ).add_to(m) + + # Click targets — small markers at gauge locations so st_folium reports clicks. + for _, row in inv.iterrows(): + nse = nse_by_gauge.get(row["gauge_id"]) + nse_text = "n/a" if (nse is None or pd.isna(nse)) else f"{nse:.3f}" + folium.CircleMarker( + location=[row["gauge_lat"], row["gauge_lon"]], + radius=5 if row["gauge_id"] == selected_gauge_id else 3, + color="#111", + weight=1.2, + fill=True, + fillColor="#111", + fillOpacity=0.85, + tooltip=folium.Tooltip( + f"{row['gauge_id']}
NSE (lead {lead}d) = {nse_text}", + sticky=True, + ), + ).add_to(m) + + cmap.add_to(m) + return m + + # --------------------------------------------------------------------------- # Plotly figures # --------------------------------------------------------------------------- -def timeseries_figure(df: pd.DataFrame, gauge_id: str) -> go.Figure: +def timeseries_figure( + df: pd.DataFrame, + gauge_id: str, + forecast: pd.DataFrame | None = None, + lead: int | None = None, +) -> go.Figure: fig = make_subplots( rows=2, cols=1, @@ -301,26 +492,91 @@ def timeseries_figure(df: pd.DataFrame, gauge_id: str) -> go.Figure: go.Scatter( x=df.index, y=df["discharge"], - name="Streamflow", - line=dict(color="#111111", width=0.9), + name="Observed Q", + line=dict(color="#111111", width=1.1), ), row=2, col=1, ) + if forecast is not None and not forecast.empty: + fig.add_trace( + go.Scatter( + x=forecast.index, + y=forecast["Q90"], + line=dict(width=0), + showlegend=False, + hoverinfo="skip", + ), + row=2, + col=1, + ) + fig.add_trace( + go.Scatter( + x=forecast.index, + y=forecast["Q10"], + fill="tonexty", + line=dict(width=0), + fillcolor="rgba(46,134,193,0.22)", + name=f"LSTM Q10–Q90 (lead {lead}d)", + hoverinfo="skip", + ), + row=2, + col=1, + ) + fig.add_trace( + go.Scatter( + x=forecast.index, + y=forecast["Q50"], + line=dict(color="#2e86c1", width=1.6), + name=f"LSTM Q50 (lead {lead}d)", + ), + row=2, + col=1, + ) # Flip precipitation axis so bars hang from the top fig.update_yaxes(autorange="reversed", title_text="P (mm/day)", row=1, col=1) - fig.update_yaxes(title_text="Q (m³/s)", row=2, col=1) + fig.update_yaxes(title_text="Q (mm/day)", row=2, col=1) fig.update_xaxes(title_text="Date", row=2, col=1) fig.update_layout( - height=420, + height=460, margin=dict(l=50, r=30, t=40, b=40), title=f"{gauge_id} — daily precipitation & streamflow", - showlegend=False, + showlegend=forecast is not None and not forecast.empty, + legend=dict(orientation="h", x=0, y=1.06, font=dict(size=10)), hovermode="x unified", ) return fig +def nse_distribution_figure(metrics: pd.DataFrame, lead: int) -> go.Figure: + sub = metrics[metrics["forecast_step"] == lead] + nse = sub["NSE"].dropna() + fig = go.Figure() + fig.add_trace( + go.Histogram( + x=nse, + xbins=dict(start=-1.0, end=1.0, size=0.05), + marker_color="#27ae60", + opacity=0.85, + ) + ) + fig.add_vline( + x=float(nse.median()), + line=dict(color="#111", dash="dash", width=1.4), + annotation_text=f"median = {float(nse.median()):.2f}", + annotation_position="top", + ) + fig.update_layout( + title=f"NSE distribution (lead {lead}d) — n={len(nse)} basins", + xaxis_title="NSE", + yaxis_title="# basins", + height=300, + margin=dict(l=40, r=20, t=50, b=40), + bargap=0.05, + ) + return fig + + def streamflow_histogram(df: pd.DataFrame) -> go.Figure: q = df["discharge"].dropna() positives = q[q > 0] @@ -549,6 +805,9 @@ def render_basin_panel( inv: pd.DataFrame, wb: pd.DataFrame, gauge_id: str, + preds: pd.DataFrame, + metrics: pd.DataFrame, + lead: int, ) -> None: basin = inv.loc[inv["gauge_id"] == gauge_id].iloc[0] wb_row = wb.loc[wb["gauge_id"] == gauge_id] @@ -569,15 +828,39 @@ def render_basin_panel( st.info("This gauge has no streamflow observations.") return + code = _code_from_gauge_id(gauge_id) + fcst = predictions_for_basin(preds, code, lead) + has_forecast = not fcst.empty + ts = load_basin_ts(base_path, region, gauge_id) - ts_plot = ts[ts.index >= pd.Timestamp("2000-01-01")] + if has_forecast: + # Zoom to the test period so the overlay is actually visible. + ts_plot = ts[(ts.index >= TEST_START) & (ts.index <= TEST_END)] + else: + ts_plot = ts[ts.index >= pd.Timestamp("2000-01-01")] if ts_plot.empty: ts_plot = ts st.plotly_chart( - timeseries_figure(ts_plot, gauge_id), + timeseries_figure(ts_plot, gauge_id, forecast=fcst, lead=lead), use_container_width=True, ) + basin_metrics = metrics[ + (metrics["gauge_id"] == gauge_id) & (metrics["forecast_step"] == lead) + ] + if basin_metrics.empty: + st.info( + f"No LSTM predictions for {gauge_id} (not in the test-set of " + f"{metrics['gauge_id'].nunique()} basins)." + ) + else: + row = basin_metrics.iloc[0] + mc = st.columns(4) + mc[0].metric(f"NSE (lead {lead}d)", f"{row['NSE']:.3f}") + mc[1].metric("nRMSE", f"{row['nRMSE']:.3f}") + mc[2].metric("nMAE", f"{row['nMAE']:.3f}") + mc[3].metric("pBias (%)", f"{row['pBias']:.2f}") + cols_rr = st.columns(2) if has_wb: rr_m3s = wb_row["RR_m3s"] @@ -636,9 +919,33 @@ def main() -> None: index=0, ) + st.markdown("### LSTM forecast") + forecast_lead = st.slider( + "Forecast lead time (days)", + min_value=1, + max_value=10, + value=1, + step=1, + help=( + "Drives both the NSE coloring on the model-performance map " + "and the Q10–Q90 uncertainty band on the timeseries plot." + ), + ) + inv = load_gauge_inventory(base_path, region) shapes = load_basin_shapes(base_path) wb = compute_water_balance(base_path, region) + run_dir = Path(CLI_ARGS.pred_dir) + if not run_dir.exists(): + st.error( + f"LSTM run dir not found: {run_dir}\n" + f"Pass a different one via `streamlit run scripts/explore_kaz.py -- " + f"--pred-dir ./path/to/run`." + ) + st.stop() + st.sidebar.caption(f"LSTM run: `{run_dir}`") + preds = load_predictions(str(run_dir)) + metrics = load_test_metrics(str(run_dir)) with st.sidebar: st.markdown("### Filters") @@ -683,12 +990,23 @@ def main() -> None: st.caption( "Tab 1 → click a basin on the map to inspect it. " - "Tab 2 → all-basin correlation plots and runoff-ratio distributions." + "Tab 2 → all-basin correlation plots and runoff-ratio distributions. " + "Tab 3 → LSTM model performance (NSE map + distribution)." ) - tab_map, tab_cross = st.tabs( - ["🗺️ Map & selected basin", "📊 Correlations & distributions (all basins)"] + tab_map, tab_cross, tab_perf = st.tabs( + [ + "🗺️ Map & selected basin", + "📊 Correlations & distributions (all basins)", + "🎯 Model performance (LSTM)", + ] ) + sel_options = inv_filt.sort_values( + ["has_streamflow", "n_streamflow_obs"], ascending=[False, False] + )["gauge_id"].tolist() + if st.session_state.selected_gauge not in sel_options and sel_options: + st.session_state.selected_gauge = sel_options[0] + with tab_map: left, right = st.columns([0.55, 0.45]) @@ -704,32 +1022,29 @@ def main() -> None: key="kaz_map", ) - sel_options = inv_filt.sort_values( - ["has_streamflow", "n_streamflow_obs"], ascending=[False, False] - )["gauge_id"].tolist() - current = st.session_state.selected_gauge - if current not in sel_options and sel_options: - current = sel_options[0] + # Handle map click BEFORE the selectbox so widget state stays in sync. + _handle_map_click(map_result, inv_filt, "_last_click_main") + sel = st.selectbox( "…or pick a gauge from the list", options=sel_options, - index=sel_options.index(current) if current in sel_options else 0, + index=sel_options.index(st.session_state.selected_gauge), key="gauge_selectbox", ) if sel != st.session_state.selected_gauge: st.session_state.selected_gauge = sel st.rerun() - if map_result and map_result.get("last_object_clicked"): - click = map_result["last_object_clicked"] - clicked = nearest_gauge(inv_filt, click["lat"], click["lng"]) - if clicked != st.session_state.selected_gauge: - st.session_state.selected_gauge = clicked - st.rerun() - with right: render_basin_panel( - base_path, region, inv, wb, st.session_state.selected_gauge + base_path, + region, + inv, + wb, + st.session_state.selected_gauge, + preds, + metrics, + forecast_lead, ) with tab_cross: @@ -737,6 +1052,57 @@ def main() -> None: with st.expander("Show water-balance table"): st.dataframe(wb.sort_values("RR_m3s"), use_container_width=True) + with tab_perf: + st.markdown( + f"Basin polygons coloured by **NSE** at forecast lead **{forecast_lead}d** " + "(RdYlGn, clamped to [-0.5, 1.0]). Grey basins have no LSTM predictions." + ) + left_p, right_p = st.columns([0.55, 0.45]) + with left_p: + m_nse = build_nse_map( + inv_filt, + shapes, + metrics, + forecast_lead, + st.session_state.selected_gauge, + ) + perf_result = st_folium( + m_nse, + width=None, + height=620, + returned_objects=["last_object_clicked"], + key="kaz_perf_map", + ) + _handle_map_click(perf_result, inv_filt, "_last_click_perf") + st.caption( + f"Selected basin: **{st.session_state.selected_gauge}** " + "(shared with the Map & selected basin tab)." + ) + + with right_p: + st.plotly_chart( + nse_distribution_figure(metrics, forecast_lead), + use_container_width=True, + ) + lead_metrics = ( + metrics[metrics["forecast_step"] == forecast_lead] + .loc[:, ["gauge_id", "NSE", "nRMSE", "nMAE", "pBias"]] + .sort_values("NSE", ascending=False) + .reset_index(drop=True) + ) + st.dataframe( + lead_metrics.style.format( + { + "NSE": "{:.3f}", + "nRMSE": "{:.3f}", + "nMAE": "{:.3f}", + "pBias": "{:.2f}", + } + ), + use_container_width=True, + height=320, + ) + if __name__ == "__main__": main() diff --git a/st_forecast/evaluation/__init__.py b/st_forecast/evaluation/__init__.py index cad0066..6c98e87 100644 --- a/st_forecast/evaluation/__init__.py +++ b/st_forecast/evaluation/__init__.py @@ -8,6 +8,12 @@ filter_agricultural_period, align_predictions_observations, ) +from .metric_functions import ( + calculate_MAE, + calculate_NSE, + calculate_PI, + calculate_RMSE, +) from .performance_metrics import ( calculate_pbias, calculate_high_flow_bias, @@ -39,6 +45,10 @@ "filter_agricultural_period", "align_predictions_observations", # Performance metrics + "calculate_NSE", + "calculate_PI", + "calculate_RMSE", + "calculate_MAE", "calculate_pbias", "calculate_high_flow_bias", "compute_basin_performance_metrics", diff --git a/st_forecast/evaluation/evaluator.py b/st_forecast/evaluation/evaluator.py index 9cceff4..91f494a 100644 --- a/st_forecast/evaluation/evaluator.py +++ b/st_forecast/evaluation/evaluator.py @@ -2,6 +2,7 @@ import logging from pathlib import Path +import numpy as np import pandas as pd from .flood_metrics import compute_all_basins_flood_metrics, save_flood_metrics @@ -87,7 +88,12 @@ def align_predictions_observations( obs_code_col: str = "code", obs_value_col: str = "discharge", ) -> pd.DataFrame: - """Merge predictions with observations on date and code.""" + """Merge predictions with observations on date and code. + + Also attaches an ``obs_at_issue`` column with the observation at the + forecast issue date (t - forecast_step days) for each row, which is + the persistence-forecast reference needed by the Persistence Index. + """ pred_df = predictions.copy() obs_df = observations[[obs_date_col, obs_code_col, obs_value_col]].copy() @@ -106,6 +112,14 @@ def align_predictions_observations( how="inner", ) + merged = _attach_obs_at_issue( + merged, + obs_df, + pred_date_col=pred_date_col, + pred_code_col=pred_code_col, + obs_value_col=obs_value_col, + ) + logger.info( f"Aligned {len(predictions)} predictions with {len(observations)} observations " f"-> {len(merged)} matched rows" @@ -113,6 +127,62 @@ def align_predictions_observations( return merged +def _attach_obs_at_issue( + merged: pd.DataFrame, + obs_df: pd.DataFrame, + pred_date_col: str, + pred_code_col: str, + obs_value_col: str, +) -> pd.DataFrame: + """Add an ``obs_at_issue`` column = observation at the forecast issue date. + + Resolves the issue date from ``forecast_date``/``forecast_issue_date`` + if present, else derives it as ``date - forecast_step days``. Rows + where the issue-date observation is missing get NaN. + """ + if len(merged) == 0: + merged = merged.copy() + merged["obs_at_issue"] = np.array([], dtype=float) + return merged + + issue_col = next((c for c in _ISSUE_DATE_CANDIDATES if c in merged.columns), None) + if issue_col is not None: + issue_dates = pd.to_datetime(merged[issue_col]) + elif "forecast_step" in merged.columns: + issue_dates = pd.to_datetime(merged[pred_date_col]) - pd.to_timedelta( + merged["forecast_step"], unit="D" + ) + else: + logger.warning( + "Cannot attach obs_at_issue: no forecast issue-date column and no " + "forecast_step. Persistence Index will be NaN." + ) + merged = merged.copy() + merged["obs_at_issue"] = np.nan + return merged + + issue_df = pd.DataFrame( + { + pred_code_col: merged[pred_code_col].values, + "_issue_date": issue_dates.values, + } + ) + + obs_issue = obs_df.rename( + columns={pred_date_col: "_issue_date", obs_value_col: "obs_at_issue"} + ) + lookup = pd.merge( + issue_df, + obs_issue, + on=[pred_code_col, "_issue_date"], + how="left", + ) + + merged = merged.copy() + merged["obs_at_issue"] = lookup["obs_at_issue"].values + return merged + + def save_all_metrics( result: EvaluationResult, output_folder: Path, diff --git a/st_forecast/evaluation/metric_functions.py b/st_forecast/evaluation/metric_functions.py index 439bafe..76babce 100644 --- a/st_forecast/evaluation/metric_functions.py +++ b/st_forecast/evaluation/metric_functions.py @@ -258,6 +258,70 @@ def calculate_NSE(observed: np.ndarray, simulated: np.ndarray) -> float: return nse +def calculate_PI( + observed: np.ndarray, + simulated: np.ndarray, + persistence: np.ndarray, +) -> float: + """ + Calculate the Persistence Index (PI) (Kitanidis & Bras, 1980). + + PI = 1 - Σ(O_t - S_t)² / Σ(O_t - O_{t-L})² + + where O_t are observed values, S_t are simulated values, and + O_{t-L} is the persistence reference (observation at the forecast + issue date, i.e. lag L = forecast_step). + + PI = 1 is a perfect forecast. PI = 0 is no better than persistence. + PI < 0 is worse than persistence. + + Parameters + ---------- + observed : np.ndarray + Array of observed values at forecast target time t. + simulated : np.ndarray + Array of simulated/forecast values at time t. + persistence : np.ndarray + Array of persistence-reference values, i.e. observed values at + time t-L (one entry per row, aligned to ``observed``). + + Returns + ------- + float + PI in (-inf, 1]. NaN if fewer than 2 jointly valid rows or if + the persistence variance is ≈ 0 (constant lagged observations). + """ + observed = np.asarray(observed) + simulated = np.asarray(simulated) + persistence = np.asarray(persistence) + + if not (observed.shape == simulated.shape == persistence.shape): + raise ValueError( + f"Observed, simulated and persistence arrays must have same shape. " + f"Got shapes {observed.shape}, {simulated.shape} and {persistence.shape}" + ) + + valid_mask = ~np.isnan(observed) & ~np.isnan(simulated) & ~np.isnan(persistence) + valid_observed = observed[valid_mask] + valid_simulated = simulated[valid_mask] + valid_persistence = persistence[valid_mask] + + if len(valid_observed) < 2: + return np.nan + + denominator = np.sum((valid_observed - valid_persistence) ** 2) + if denominator < 1e-10: + return np.nan + + numerator = np.sum((valid_observed - valid_simulated) ** 2) + pi = 1 - (numerator / denominator) + + if np.isinf(pi) or np.isnan(pi): + return np.nan + + return pi + + def calculate_RMSE( observed: np.ndarray, simulated: np.ndarray, normalize: bool = True ) -> Union[float, Tuple[float, int]]: diff --git a/st_forecast/evaluation/performance_metrics.py b/st_forecast/evaluation/performance_metrics.py index 2e9328c..ef394fc 100644 --- a/st_forecast/evaluation/performance_metrics.py +++ b/st_forecast/evaluation/performance_metrics.py @@ -4,7 +4,12 @@ import numpy as np import pandas as pd -from .metric_functions import calculate_MAE, calculate_NSE, calculate_RMSE +from .metric_functions import ( + calculate_MAE, + calculate_NSE, + calculate_PI, + calculate_RMSE, +) logger = logging.getLogger(__name__) @@ -71,13 +76,24 @@ def compute_basin_performance_metrics( merged_df: pd.DataFrame, obs_col: str = "discharge", sim_col: str = "Q50", + persistence_col: str = "obs_at_issue", ) -> dict[str, float]: - """Compute all metrics for a single basin.""" + """Compute all metrics for a single basin. + + If ``persistence_col`` is present, also computes the Persistence Index + (PI = NSE vs. persistence baseline). Otherwise PI is set to NaN. + """ observed = merged_df[obs_col].values simulated = merged_df[sim_col].values + if persistence_col in merged_df.columns: + pi = calculate_PI(observed, simulated, merged_df[persistence_col].values) + else: + pi = np.nan + return { "NSE": calculate_NSE(observed, simulated), + "PI": pi, "nRMSE": calculate_RMSE(observed, simulated, normalize=True), "nMAE": calculate_MAE(observed, simulated, normalize=True), "pBias": calculate_pbias(observed, simulated), @@ -111,6 +127,7 @@ def compute_all_basins_performance( "code", "forecast_step", "NSE", + "PI", "nRMSE", "nMAE", "pBias", diff --git a/st_forecast/visualization/compare_runs.py b/st_forecast/visualization/compare_runs.py index 6815fa4..e553ced 100644 --- a/st_forecast/visualization/compare_runs.py +++ b/st_forecast/visualization/compare_runs.py @@ -137,6 +137,118 @@ def plot_nse_comparison( return None +def plot_pi_comparison( + config: RunComparisonConfig, + output_path: Path | None = None, + floor: float = -1.0, +) -> Path | None: + """Boxplot of Persistence Index per forecast step, across runs. + + PI = 1 means perfect; PI = 0 means no better than persistence; PI < 0 + means worse than persistence. A reference line is drawn at PI = 0 and + the y-axis is floored at ``floor`` so very poor runs don't squash the + visual range; basins below 0 (worse than persistence) are annotated. + """ + _apply_conference_style() + + data = load_multiple_run_metrics( + config.run_folders, config.run_labels, config.split + ) + metrics_df = data.get("metrics", pd.DataFrame()) + + if metrics_df.empty or "PI" not in metrics_df.columns: + logger.warning("No PI metrics available for comparison") + return None + + if "forecast_step" not in metrics_df.columns: + logger.warning("No forecast_step column in metrics") + return None + + palette = _build_palette(config.run_labels) + + plot_df = metrics_df.copy() + plot_df["PI_clipped"] = plot_df["PI"].clip(lower=floor) + + fig, ax = plt.subplots(figsize=(12, 6)) + + sns.boxplot( + data=plot_df, + x="forecast_step", + y="PI_clipped", + hue="run", + palette=palette, + ax=ax, + showfliers=False, + boxprops={"alpha": 0.4}, + whiskerprops={"alpha": 0.4}, + capprops={"alpha": 0.4}, + medianprops={"alpha": 0.8}, + ) + + sns.stripplot( + data=plot_df, + x="forecast_step", + y="PI_clipped", + hue="run", + palette=palette, + ax=ax, + dodge=True, + alpha=0.4, + size=3, + legend=False, + ) + + ax.axhline(0.0, color="black", linewidth=1.0, linestyle="--", alpha=0.6) + + ax.set_xlabel("Forecast Step [days]") + ax.set_ylabel("PI [-]") + ax.set_title("") + ax.set_ylim(floor, 1.0) + ax.legend(title=None, bbox_to_anchor=(1.02, 1), loc="upper left") + + # Count basins worse than persistence (PI < 0) per run and forecast step + below_counts = ( + metrics_df[metrics_df["PI"] < 0.0] + .groupby(["forecast_step", "run"]) + .size() + .reset_index(name="n_below") + ) + + if not below_counts.empty: + steps = sorted(metrics_df["forecast_step"].unique()) + runs = [lbl for lbl in config.run_labels if lbl in metrics_df["run"].unique()] + n_runs = len(runs) + + for _, row in below_counts.iterrows(): + if row["run"] not in runs: + continue + step_idx = steps.index(row["forecast_step"]) + run_idx = runs.index(row["run"]) + width = 0.8 / n_runs + offset = (run_idx - (n_runs - 1) / 2) * width + ax.text( + step_idx + offset, + floor + 0.02, + str(row["n_below"]), + ha="center", + va="bottom", + fontsize=7, + color=palette.get(row["run"], "black"), + fontweight="bold", + ) + + plt.tight_layout() + + if output_path is not None: + output_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_path, dpi=300, bbox_inches="tight") + plt.close(fig) + return output_path + + plt.close(fig) + return None + + def plot_nrmse_comparison( config: RunComparisonConfig, output_path: Path | None = None, @@ -454,6 +566,7 @@ def generate_comparison_plots(config: RunComparisonConfig) -> dict[str, Path]: plot_functions = [ ("nse_comparison", plot_nse_comparison), + ("pi_comparison", plot_pi_comparison), ("nrmse_comparison", plot_nrmse_comparison), ("calibration_comparison", plot_calibration_comparison), ("crps_comparison", plot_crps_comparison), diff --git a/st_forecast/visualization/metrics.py b/st_forecast/visualization/metrics.py index 63d6ad9..d0de5fd 100644 --- a/st_forecast/visualization/metrics.py +++ b/st_forecast/visualization/metrics.py @@ -17,6 +17,7 @@ # Metric display configuration _METRIC_LABELS: dict[str, tuple[str, str]] = { "NSE": ("NSE [-]", "Nash-Sutcliffe Efficiency (NSE)"), + "PI": ("PI [-]", "Persistence Index (PI)"), "RMSE": ("RMSE [m\u00b3/s]", "Root Mean Square Error (RMSE)"), "MAE": ("MAE [m\u00b3/s]", "Mean Absolute Error (MAE)"), "nRMSE": ("nRMSE [-]", "Normalized Root Mean Square Error (nRMSE)"), diff --git a/tests/test_evaluation/test_evaluator.py b/tests/test_evaluation/test_evaluator.py index 801c037..7676518 100644 --- a/tests/test_evaluation/test_evaluator.py +++ b/tests/test_evaluation/test_evaluator.py @@ -1,16 +1,18 @@ +import numpy as np import pandas as pd import pytest -from st_forecast.evaluation.evaluator import ensure_forecast_step +from st_forecast.evaluation.evaluator import ( + align_predictions_observations, + ensure_forecast_step, +) class TestEnsureForecastStep: def test_computes_from_forecast_date(self): df = pd.DataFrame( { - "date": pd.to_datetime( - ["2024-01-02", "2024-01-03", "2024-01-04"] - ), + "date": pd.to_datetime(["2024-01-02", "2024-01-03", "2024-01-04"]), "forecast_date": pd.to_datetime( ["2024-01-01", "2024-01-01", "2024-01-01"] ), @@ -26,12 +28,8 @@ def test_computes_from_forecast_date(self): def test_computes_from_forecast_issue_date(self): df = pd.DataFrame( { - "date": pd.to_datetime( - ["2024-01-05", "2024-01-07"] - ), - "forecast_issue_date": pd.to_datetime( - ["2024-01-04", "2024-01-04"] - ), + "date": pd.to_datetime(["2024-01-05", "2024-01-07"]), + "forecast_issue_date": pd.to_datetime(["2024-01-04", "2024-01-04"]), "code": [1, 1], "Q50": [10.0, 11.0], } @@ -81,3 +79,92 @@ def test_accepts_string_date_columns(self): result = ensure_forecast_step(df) assert list(result["forecast_step"]) == [1, 3] + + +class TestAlignPredictionsObservationsObsAtIssue: + def test_attaches_obs_at_issue_from_forecast_date(self): + """obs_at_issue should be Q_obs at the forecast issue date.""" + predictions = pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-02", "2024-01-03", "2024-01-04"]), + "forecast_date": pd.to_datetime( + ["2024-01-01", "2024-01-01", "2024-01-01"] + ), + "code": [1, 1, 1], + "Q50": [10.0, 11.0, 12.0], + } + ) + observations = pd.DataFrame( + { + "date": pd.to_datetime( + [ + "2024-01-01", + "2024-01-02", + "2024-01-03", + "2024-01-04", + ] + ), + "code": [1, 1, 1, 1], + "discharge": [5.0, 6.0, 7.0, 8.0], + } + ) + + merged = align_predictions_observations(predictions, observations) + + assert "obs_at_issue" in merged.columns + # All three forecast rows were issued on 2024-01-01 → obs_at_issue=5.0 + assert list(merged["obs_at_issue"]) == [5.0, 5.0, 5.0] + + def test_derives_issue_date_from_forecast_step(self): + """When only forecast_step is present, derive issue date as date-step.""" + predictions = pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-04", "2024-01-05"]), + "forecast_step": [3, 2], + "code": [1, 1], + "Q50": [10.0, 11.0], + } + ) + observations = pd.DataFrame( + { + "date": pd.to_datetime( + [ + "2024-01-01", + "2024-01-03", + "2024-01-04", + "2024-01-05", + ] + ), + "code": [1, 1, 1, 1], + "discharge": [100.0, 300.0, 400.0, 500.0], + } + ) + + merged = align_predictions_observations(predictions, observations) + + # Row 0: date 01-04 - 3d = 01-01 → 100.0 + # Row 1: date 01-05 - 2d = 01-03 → 300.0 + assert list(merged["obs_at_issue"]) == [100.0, 300.0] + + def test_missing_issue_obs_yields_nan(self): + """If the issue-date observation is missing, obs_at_issue is NaN.""" + predictions = pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-02"]), + "forecast_date": pd.to_datetime(["2024-01-01"]), + "code": [1], + "Q50": [10.0], + } + ) + observations = pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-02"]), # no 01-01 row + "code": [1], + "discharge": [12.0], + } + ) + + merged = align_predictions_observations(predictions, observations) + + assert "obs_at_issue" in merged.columns + assert np.isnan(merged["obs_at_issue"].iloc[0]) diff --git a/tests/test_evaluation/test_performance_metrics.py b/tests/test_evaluation/test_performance_metrics.py index e5863e7..46047c5 100644 --- a/tests/test_evaluation/test_performance_metrics.py +++ b/tests/test_evaluation/test_performance_metrics.py @@ -4,6 +4,7 @@ from pathlib import Path import tempfile +from st_forecast.evaluation.metric_functions import calculate_PI from st_forecast.evaluation.performance_metrics import ( calculate_pbias, calculate_high_flow_bias, @@ -141,6 +142,82 @@ def test_shape_mismatch_raises_error(self): calculate_high_flow_bias(obs, sim) +class TestCalculatePI: + def test_perfect_forecast_returns_one(self): + """sim == obs (model recovers truth) should give PI = 1.""" + obs = np.array([10.0, 12.0, 15.0, 9.0, 11.0]) + sim = obs.copy() + persistence = np.array([9.0, 10.0, 12.0, 15.0, 9.0]) + assert calculate_PI(obs, sim, persistence) == pytest.approx(1.0) + + def test_model_equals_persistence_returns_zero(self): + """When sim equals the persistence reference, PI should be 0.""" + obs = np.array([10.0, 12.0, 15.0, 9.0, 11.0]) + persistence = np.array([9.0, 10.0, 12.0, 15.0, 9.0]) + sim = persistence.copy() + assert calculate_PI(obs, sim, persistence) == pytest.approx(0.0) + + def test_model_worse_than_persistence_returns_negative(self): + """When sim errors exceed persistence errors, PI should be < 0.""" + obs = np.array([10.0, 12.0, 15.0, 9.0, 11.0]) + persistence = np.array([10.5, 12.5, 14.5, 9.5, 10.5]) # off by ~0.5 + sim = np.array([20.0, 0.0, 25.0, 0.0, 20.0]) # wildly wrong + result = calculate_PI(obs, sim, persistence) + assert result < 0.0 + + def test_constant_persistence_returns_nan(self): + """Zero variance in (obs - persistence) makes the denominator ~0.""" + obs = np.array([10.0, 10.0, 10.0, 10.0]) + persistence = np.array([10.0, 10.0, 10.0, 10.0]) + sim = np.array([11.0, 9.0, 11.0, 9.0]) + result = calculate_PI(obs, sim, persistence) + assert np.isnan(result) + + def test_handles_nan_in_any_array(self): + """NaN in obs, sim, or persistence should drop the row jointly.""" + obs = np.array([10.0, np.nan, 15.0, 9.0, 11.0]) + sim = np.array([10.0, 12.0, np.nan, 9.0, 11.0]) + persistence = np.array([9.0, 10.0, 12.0, np.nan, 9.0]) + # Only rows 0 and 4 are jointly valid: + # obs=[10, 11], sim=[10, 11], persistence=[9, 9] + # sim == obs ⇒ numerator = 0 ⇒ PI = 1 + assert calculate_PI(obs, sim, persistence) == pytest.approx(1.0) + + def test_insufficient_data_returns_nan(self): + """Fewer than 2 jointly valid rows should yield NaN.""" + obs = np.array([10.0, np.nan]) + sim = np.array([10.0, 12.0]) + persistence = np.array([9.0, 10.0]) + result = calculate_PI(obs, sim, persistence) + assert np.isnan(result) + + def test_shape_mismatch_raises_error(self): + """Mismatched array shapes should raise ValueError.""" + obs = np.array([10.0, 20.0, 30.0]) + sim = np.array([10.0, 20.0]) + persistence = np.array([9.0, 10.0, 12.0]) + with pytest.raises(ValueError, match="same shape"): + calculate_PI(obs, sim, persistence) + + def test_accepts_lists(self): + """Should accept lists and convert to arrays.""" + obs = [10.0, 12.0, 15.0] + sim = [10.0, 12.0, 15.0] + persistence = [9.0, 10.0, 12.0] + assert calculate_PI(obs, sim, persistence) == pytest.approx(1.0) + + def test_formula_matches_kitanidis_bras(self): + """Spot-check the closed-form value against a hand calculation.""" + obs = np.array([10.0, 20.0, 30.0, 40.0]) + sim = np.array([12.0, 18.0, 33.0, 38.0]) + persistence = np.array([8.0, 10.0, 20.0, 30.0]) + # numerator = 4 + 4 + 9 + 4 = 21 + # denominator = 4 + 100 + 100 + 100 = 304 + # PI = 1 - 21/304 ≈ 0.93092 + expected = 1.0 - 21.0 / 304.0 + assert calculate_PI(obs, sim, persistence) == pytest.approx(expected) + + class TestComputeBasinPerformanceMetrics: def test_computes_all_metrics(self): """Should compute NSE, nRMSE, nMAE, pBias, and high_flow_bias.""" @@ -182,6 +259,29 @@ def test_handles_missing_values(self): # Should compute metrics only on valid pairs: (10, 10) assert metrics["pBias"] == pytest.approx(0.0) + def test_pi_present_when_obs_at_issue_provided(self): + """PI should be computed when obs_at_issue column is available.""" + df = pd.DataFrame( + { + "discharge": [10.0, 12.0, 15.0, 9.0, 11.0], + "Q50": [10.0, 12.0, 15.0, 9.0, 11.0], + "obs_at_issue": [9.0, 10.0, 12.0, 15.0, 9.0], + } + ) + metrics = compute_basin_performance_metrics(df) + + assert "PI" in metrics + # Perfect forecast → PI = 1.0 + assert metrics["PI"] == pytest.approx(1.0) + + def test_pi_is_nan_without_obs_at_issue(self): + """PI should be NaN when obs_at_issue column is missing.""" + df = pd.DataFrame({"discharge": [10.0, 20.0, 30.0], "Q50": [10.0, 20.0, 30.0]}) + metrics = compute_basin_performance_metrics(df) + + assert "PI" in metrics + assert np.isnan(metrics["PI"]) + class TestComputeAllBasinsPerformance: def test_computes_metrics_per_basin_and_step(self): From d95c4d5681a7b8f0d3446a0f046085fe47648939 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:30:52 +0200 Subject: [PATCH 13/20] mse loss --- scripts/compare_runs.py | 16 ++++++++++++---- scripts/explore_kaz.py | 2 +- st_forecast/model_helper.py | 5 +++-- st_forecast/pipeline/artifacts.py | 2 +- st_forecast/training_components.py | 2 ++ 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/scripts/compare_runs.py b/scripts/compare_runs.py index 3b54e8f..75e3cd3 100644 --- a/scripts/compare_runs.py +++ b/scripts/compare_runs.py @@ -13,7 +13,17 @@ # CONFIGURATION - Edit these variables # ============================================================================= + RUN_FOLDERS: list[Path] = [ + Path("./runs/kaz/lstm"), + Path("./runs/kaz/tsmixer"), + Path("./runs/kaz/tide"), + Path("./runs/kaz/tft"), + # Add more runs as needed +] + + +"""RUN_FOLDERS: list[Path] = [ Path("./runs/persistence_kgz"), Path("./runs/egu/kgz_lstm"), Path("./runs/egu/tsmixer_kgz"), @@ -21,7 +31,7 @@ Path("./runs/egu/tft_kgz"), Path("./runs/ensemble_kgz"), # Add more runs as needed -] +]""" """RUN_FOLDERS: list[Path] = [ Path("./runs/persistence_taj"), @@ -42,15 +52,13 @@ ] RUN_LABELS: list[str] = [ - "Persistence", "LSTM", "TSMixer", "TiDE", "TFT", - "Ensemble", ] -OUTPUT_DIR: Path = Path("comparison_output/kgz") +OUTPUT_DIR: Path = Path("comparison_output/kaz") SPLIT: str = "test" # "train", "val", or "test" diff --git a/scripts/explore_kaz.py b/scripts/explore_kaz.py index b746f2e..6666b72 100644 --- a/scripts/explore_kaz.py +++ b/scripts/explore_kaz.py @@ -45,7 +45,7 @@ "KAZ_DATA/sandro-basins-kaz" ) DEFAULT_REGION = "kaz" -DEFAULT_PRED_DIR = "runs/kaz/lstm" +DEFAULT_PRED_DIR = "runs/kaz/tft" TEST_START = pd.Timestamp("2016-01-02") TEST_END = pd.Timestamp("2020-12-30") QUANTILE_COLS = ["Q5", "Q10", "Q25", "Q50", "Q75", "Q90", "Q95"] diff --git a/st_forecast/model_helper.py b/st_forecast/model_helper.py index 2601d18..cb21834 100644 --- a/st_forecast/model_helper.py +++ b/st_forecast/model_helper.py @@ -171,8 +171,9 @@ def _get_model_specific_kwargs(config: RunConfig) -> dict: extra = config.extra_hyperparams # RIN is incompatible with probabilistic likelihoods (except quantile regression) - # because RIN can produce invalid scale parameters for distributions - use_rin = config.likelihood_type == "quantile" + # because RIN can produce invalid scale parameters for distributions. + # MSE is a point predictor and is also safe to combine with RIN. + use_rin = config.likelihood_type in {"quantile", "mse"} match config.model_type: case "TFT": diff --git a/st_forecast/pipeline/artifacts.py b/st_forecast/pipeline/artifacts.py index 8198bc2..0d06a78 100644 --- a/st_forecast/pipeline/artifacts.py +++ b/st_forecast/pipeline/artifacts.py @@ -110,7 +110,7 @@ class RunConfig: early_stopping_enabled: bool = True early_stopping_min_delta: float = 0.001 likelihood_type: str = ( - "quantile" # "quantile" | "gaussian" | "gamma" | "gumbel" | "laplace" + "quantile" # "quantile" | "gaussian" | "gamma" | "gumbel" | "laplace" | "cauchy" | "mse" ) gradient_clip_val: float = 0.1 accelerator: str = "auto" diff --git a/st_forecast/training_components.py b/st_forecast/training_components.py index b712636..9277f79 100644 --- a/st_forecast/training_components.py +++ b/st_forecast/training_components.py @@ -80,6 +80,8 @@ def create_likelihood(likelihood_type: str, quantiles: list[float]): return LaplaceLikelihood() case "cauchy": return CauchyLikelihood() + case "mse": + return None case _: raise ValueError(f"Unknown likelihood_type: {likelihood_type}") From da52bdbade377085b4ba6f6d9ae321e812597d6b Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:02:50 +0200 Subject: [PATCH 14/20] kaz explorer --- pyproject.toml | 1 + scripts/compare_runs.py | 2 + scripts/create_ensemble.py | 54 +++----- scripts/explore_kaz.py | 111 +++++++++++++++ st_forecast/visualization/compare_runs.py | 8 +- uv.lock | 162 +++++++++++++++++++++- 6 files changed, 290 insertions(+), 48 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6291057..7184aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ dev = [ "plotly>=6.7.0", "pytest>=8.4.2", "ruff>=0.13.2", + "selenium>=4.44.0", "streamlit>=1.57.0", "streamlit-folium>=0.27.2", ] diff --git a/scripts/compare_runs.py b/scripts/compare_runs.py index 75e3cd3..9b01eea 100644 --- a/scripts/compare_runs.py +++ b/scripts/compare_runs.py @@ -19,6 +19,7 @@ Path("./runs/kaz/tsmixer"), Path("./runs/kaz/tide"), Path("./runs/kaz/tft"), + Path("./runs/kaz/ensemble"), # Add more runs as needed ] @@ -56,6 +57,7 @@ "TSMixer", "TiDE", "TFT", + "Ensemble", ] OUTPUT_DIR: Path = Path("comparison_output/kaz") diff --git a/scripts/create_ensemble.py b/scripts/create_ensemble.py index 64c1c8c..4126975 100644 --- a/scripts/create_ensemble.py +++ b/scripts/create_ensemble.py @@ -18,9 +18,9 @@ import numpy as np import pandas as pd -from st_forecast.data_utils.loaders import CaravanDataCollector, RawDataCollector +from st_forecast.data_utils.loaders import build_collector from st_forecast.evaluation.evaluator import EvaluationConfig, evaluate -from st_forecast.pipeline.artifacts import RunConfig, load_run_artifacts +from st_forecast.pipeline.artifacts import load_run_artifacts from st_forecast.pipeline.inference import save_predictions from st_forecast.visualization import ( VisualizationConfig, @@ -45,7 +45,14 @@ Path("./runs/egu/kgz_lstm"), ] -OUTPUT_DIR: Path = Path("./runs/ensemble_kgz") +ENSEMBLE_MEMBERS: list[Path] = [ + Path("./runs/kaz/tsmixer"), + Path("./runs/kaz/tide"), + Path("./runs/kaz/tft"), + Path("./runs/kaz/lstm"), +] + +OUTPUT_DIR: Path = Path("./runs/kaz/ensemble") SPLIT: str = "test" @@ -251,43 +258,14 @@ def load_and_align_predictions( return common_keys, model_quantiles -def build_paths_dict(config: RunConfig) -> dict: - return { - "path_rivers": config.path_rivers, - "path_forcing": config.path_forcing, - "path_to_operational_forcing": config.path_to_operational_forcing, - "path_to_hindcast_forcing": config.path_to_hindcast_forcing, - "HRU_forcing": config.HRU_forcing, - "path_static": config.path_static, - "path_to_sla": config.path_to_sla, - "path_to_nir": config.path_to_nir, - "path_to_swe": config.path_to_swe, - "path_to_hs": config.path_to_hs, - "path_to_rof": config.path_to_rof, - } - - def load_observations(member_folder: Path) -> pd.DataFrame: - """Load observations using config from an ensemble member.""" - run_config, _ = load_run_artifacts(member_folder) + """Load observations using config from an ensemble member. - if getattr(run_config, "data_source", "legacy") == "caravan" and run_config.caravan: - from st_forecast.config.data_config import CaravanConfig - - caravan_config = CaravanConfig( - base_path=run_config.caravan["base_path"], - region=run_config.caravan["region"], - n_basins=run_config.caravan.get("n_basins"), - basin_ids=run_config.caravan.get("basin_ids"), - start_date=run_config.caravan.get("start_date"), - end_date=run_config.caravan.get("end_date"), - ) - collector = CaravanDataCollector(caravan_config) - else: - paths_dict = build_paths_dict(run_config) - collector = RawDataCollector(paths_dict, run_config.region) - - return collector.collect_discharge() + Dispatches to the right collector (legacy / caravan / hive) via + ``build_collector`` so all data sources are handled consistently. + """ + run_config, _ = load_run_artifacts(member_folder) + return build_collector(run_config).collect_discharge() # ============================================================================= diff --git a/scripts/explore_kaz.py b/scripts/explore_kaz.py index 6666b72..26967a5 100644 --- a/scripts/explore_kaz.py +++ b/scripts/explore_kaz.py @@ -19,6 +19,7 @@ import argparse import logging +import time from pathlib import Path import branca.colormap as bcm @@ -459,6 +460,111 @@ def style_fn(feat: dict) -> dict: return m +# --------------------------------------------------------------------------- +# Map PNG export (headless browser) +# --------------------------------------------------------------------------- + + +def _build_headless_driver(width: int, height: int, scale: float): + """Build a headless Selenium driver, preferring Chrome then Firefox. + + `scale` is the device pixel ratio — values >1 produce a higher-resolution + screenshot at the same layout. Selenium 4's driver manager auto-resolves + chromedriver/geckodriver, so no manual driver install is needed as long as + Chrome or Firefox is present. + """ + from selenium import webdriver + + try: + opts = webdriver.chrome.options.Options() + opts.add_argument("--headless=new") + opts.add_argument("--hide-scrollbars") + opts.add_argument(f"--window-size={width},{height}") + opts.add_argument(f"--force-device-scale-factor={scale}") + return webdriver.Chrome(options=opts) + except Exception as chrome_err: # noqa: BLE001 — fall back to Firefox + logger.info("Chrome unavailable (%s); trying Firefox", chrome_err) + opts = webdriver.firefox.options.Options() + opts.add_argument("--headless") + opts.add_argument(f"--width={width}") + opts.add_argument(f"--height={height}") + opts.set_preference("layout.css.devPixelsPerPx", str(scale)) + return webdriver.Firefox(options=opts) + + +@st.cache_data(show_spinner="Rendering map to PNG…") +def render_map_png( + map_html: str, + width: int = 1600, + height: int = 1050, + scale: float = 2.0, + delay: int = 4, +) -> bytes: + """Render a folium map's HTML to PNG bytes via a headless browser. + + Cached on the rendered HTML string, so re-downloading an unchanged map is + instant. `width`/`height` set the map extent; `scale` is the device pixel + ratio (>1 → higher resolution); `delay` gives basemap tiles time to load. + """ + from folium.utilities import temp_html_filepath + + driver = _build_headless_driver(width, height, scale) + try: + with temp_html_filepath(map_html) as fname: + driver.get(f"file:///{fname}") + time.sleep(delay) + _dedupe_map_legends(driver) + element = driver.find_element("class name", "folium-map") + return element.screenshot_as_png + finally: + driver.quit() + + +# branca's color_scale.js *appends* the colorbar SVG to ``.legend.leaflet-control`` +# (rather than replacing it), so a colorbar can end up drawn more than once. Keep +# only the first legend control and its first SVG so exactly one colorbar shows. +_DEDUPE_LEGENDS_JS = """ +const controls = document.querySelectorAll('.legend.leaflet-control'); +controls.forEach((ctrl, i) => { + if (i > 0) { ctrl.remove(); return; } + ctrl.querySelectorAll('svg').forEach((svg, j) => { if (j > 0) svg.remove(); }); +}); +""" + + +def _dedupe_map_legends(driver) -> None: + """Remove duplicate branca colorbar legends in the rendered page.""" + driver.execute_script(_DEDUPE_LEGENDS_JS) + + +def map_png_download(m: folium.Map, ui_key: str, filename: str) -> None: + """Render a 'Download map as PNG' control for a folium map. + + Rendering spins up a headless browser (a few seconds), so it's gated behind + a button rather than run on every Streamlit rerun. The result is held in + session state so the download button persists across reruns. + """ + state_key = f"_map_png_{ui_key}" + if st.button("🖼️ Render map as PNG", key=f"btn_{ui_key}"): + try: + st.session_state[state_key] = render_map_png(m.get_root().render()) + except Exception as err: # noqa: BLE001 — surface to the UI + st.session_state[state_key] = None + st.error( + f"PNG export failed: {err}\n\n" + "Needs a headless Chrome or Firefox available on this machine." + ) + png = st.session_state.get(state_key) + if png: + st.download_button( + "⬇️ Download PNG", + data=png, + file_name=filename, + mime="image/png", + key=f"dl_{ui_key}", + ) + + # --------------------------------------------------------------------------- # Plotly figures # --------------------------------------------------------------------------- @@ -1022,6 +1128,8 @@ def main() -> None: key="kaz_map", ) + map_png_download(m, "main", "kaz_map.png") + # Handle map click BEFORE the selectbox so widget state stays in sync. _handle_map_click(map_result, inv_filt, "_last_click_main") @@ -1073,6 +1181,9 @@ def main() -> None: returned_objects=["last_object_clicked"], key="kaz_perf_map", ) + map_png_download( + m_nse, "perf", f"kaz_nse_map_lead{forecast_lead}d.png" + ) _handle_map_click(perf_result, inv_filt, "_last_click_perf") st.caption( f"Selected basin: **{st.session_state.selected_gauge}** " diff --git a/st_forecast/visualization/compare_runs.py b/st_forecast/visualization/compare_runs.py index e553ced..5a25e8b 100644 --- a/st_forecast/visualization/compare_runs.py +++ b/st_forecast/visualization/compare_runs.py @@ -92,12 +92,12 @@ def plot_nse_comparison( ax.set_xlabel("Forecast Step [days]") ax.set_ylabel("NSE [-]") ax.set_title("") - ax.set_ylim(0.4, 1.0) + ax.set_ylim(0.0, 1.0) ax.legend(title=None, bbox_to_anchor=(1.02, 1), loc="upper left") - # Count basins below 0.4 per model and forecast step + # Count basins below 0 per model and forecast step below_counts = ( - metrics_df[metrics_df["NSE"] < 0.4] + metrics_df[metrics_df["NSE"] < 0.0] .groupby(["forecast_step", "run"]) .size() .reset_index(name="n_below") @@ -116,7 +116,7 @@ def plot_nse_comparison( offset = (run_idx - (n_runs - 1) / 2) * width ax.text( step_idx + offset, - 0.405, + 0.005, str(row["n_below"]), ha="center", va="bottom", diff --git a/uv.lock b/uv.lock index 4157b16..f0552fb 100644 --- a/uv.lock +++ b/uv.lock @@ -255,11 +255,37 @@ wheels = [ [[package]] name = "certifi" -version = "2025.8.3" +version = "2026.5.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] [[package]] @@ -1679,6 +1705,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/12/cba81286cbaf0f0c3f0473846cfd992cb240bdcea816bf2ef7de8ed0f744/optuna-4.5.0-py3-none-any.whl", hash = "sha256:5b8a783e84e448b0742501bc27195344a28d2c77bd2feef5b558544d954851b0", size = 400872, upload-time = "2025-08-18T06:49:20.697Z" }, ] +[[package]] +name = "outcome" +version = "1.3.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" }, +] + [[package]] name = "packaging" version = "24.2" @@ -2033,6 +2071,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -2330,6 +2377,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/06/cad54e8ce758bd836ee5411691cbd49efeb9cc611b374670fce299519334/pyshp-3.0.3-py3-none-any.whl", hash = "sha256:28c8fac8c0c25bb0fecbbfd10ead7f319c2ff2f3b0b44a94f22bd2c93510ad42", size = 58465, upload-time = "2025-11-28T17:47:30.328Z" }, ] +[[package]] +name = "pysocks" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, +] + [[package]] name = "pytest" version = "8.4.2" @@ -2892,6 +2948,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, ] +[[package]] +name = "selenium" +version = "4.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "trio" }, + { name = "trio-websocket" }, + { name = "typing-extensions" }, + { name = "urllib3", extra = ["socks"] }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/4a/6d0a4f4a07e2a91511a51398203ee82bf6ce644a448aaa35c59b44aa9531/selenium-4.44.0.tar.gz", hash = "sha256:b03a831fcfcab9d912b4682f60718c48a04560d6c62f7496c16b7498c9a4427e", size = 993133, upload-time = "2026-05-12T22:48:19.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/bc/885047e975e996cb317db31c4551caa915aafc6befea990f082c7233adc2/selenium-4.44.0-py3-none-any.whl", hash = "sha256:d01ea3e5ecad8149460a765f7cf5177194c21dcc0173093fc05427c289b1bf24", size = 9654291, upload-time = "2026-05-12T22:48:16.836Z" }, +] + [[package]] name = "setuptools" version = "80.9.0" @@ -3025,6 +3098,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.43" @@ -3096,6 +3187,7 @@ dev = [ { name = "plotly" }, { name = "pytest" }, { name = "ruff" }, + { name = "selenium" }, { name = "streamlit" }, { name = "streamlit-folium" }, ] @@ -3131,6 +3223,7 @@ dev = [ { name = "plotly", specifier = ">=6.7.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "ruff", specifier = ">=0.13.2" }, + { name = "selenium", specifier = ">=4.44.0" }, { name = "streamlit", specifier = ">=1.57.0" }, { name = "streamlit-folium", specifier = ">=0.27.2" }, ] @@ -3385,6 +3478,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, ] +[[package]] +name = "trio" +version = "0.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" }, + { name = "idna" }, + { name = "outcome" }, + { name = "sniffio" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" }, +] + +[[package]] +name = "trio-websocket" +version = "0.12.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "outcome" }, + { name = "trio" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/3c/8b4358e81f2f2cfe71b66a267f023a91db20a817b9425dd964873796980a/trio_websocket-0.12.2.tar.gz", hash = "sha256:22c72c436f3d1e264d0910a3951934798dcc5b00ae56fc4ee079d46c7cf20fae", size = 33549, upload-time = "2025-02-25T05:16:58.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/19/eb640a397bba49ba49ef9dbe2e7e5c04202ba045b6ce2ec36e9cadc51e04/trio_websocket-0.12.2-py3-none-any.whl", hash = "sha256:df605665f1db533f4a386c94525870851096a223adcb97f72a07e8b4beba45b6", size = 21221, upload-time = "2025-02-25T05:16:57.545Z" }, +] + [[package]] name = "triton" version = "3.4.0" @@ -3431,11 +3555,16 @@ wheels = [ [[package]] name = "urllib3" -version = "2.5.0" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[package.optional-dependencies] +socks = [ + { name = "pysocks" }, ] [[package]] @@ -3490,6 +3619,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "16.0" @@ -3549,6 +3687,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, ] +[[package]] +name = "wsproto" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, +] + [[package]] name = "xarray" version = "2025.9.1" From 172eb274dc2474fdc6c3e34bd35b6ea0ff712adf Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:55:18 +0200 Subject: [PATCH 15/20] kaz export --- .gitignore | 3 + scripts/build_kaz_export.py | 268 ++++++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+) create mode 100644 scripts/build_kaz_export.py diff --git a/.gitignore b/.gitignore index 7bd8863..304106f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ __pycache__/ /st_forecast/data_utils/__pycache__ /comparison_output /sapphire_ref +/intermediate_results_kaz +/memory +/kaz_export diff --git a/scripts/build_kaz_export.py b/scripts/build_kaz_export.py new file mode 100644 index 0000000..0928c7d --- /dev/null +++ b/scripts/build_kaz_export.py @@ -0,0 +1,268 @@ +"""Assemble the self-contained KAZ dashboard export package. + +Builds `intermediate_results_kaz/` (gitignored) from the live model runs and the +Hive basin dataset, so it can be zipped and handed to Kazakh colleagues. + +Run with: + + uv run python scripts/build_kaz_export.py + uv run python scripts/build_kaz_export.py --hive-base /path/to/sandro-basins-kaz + +Produces: + intermediate_results_kaz/ + kaz_dashboard.py, requirements.txt, README.md (copied from kaz_export/) + runs//{config.json, static_df.csv, predictions/, metrics/} + data/stations.csv + data/observations/kaz_.csv (date, discharge_mm_per_day, P_mm_per_day) + data/updated_watersheds.* (shapefile for the maps) + predictions_csv//lead_/kaz_.csv (9 cols, 2 dp, both units) +""" + +from __future__ import annotations + +import argparse +import logging +import shutil +from pathlib import Path + +import pandas as pd + +from st_forecast.data_utils.hive_loaders import _attributes_file, _timeseries_dir + +logging.basicConfig(level=logging.INFO, format="%(levelname)s | %(message)s") +logger = logging.getLogger("build_kaz_export") + +REPO_ROOT = Path(__file__).resolve().parent.parent +RUNS_SRC = REPO_ROOT / "runs" / "kaz" +TEMPLATE_DIR = REPO_ROOT / "kaz_export" +OUT_DIR = REPO_ROOT / "intermediate_results_kaz" + +DEFAULT_HIVE_BASE = ( + "/Users/sandrohunziker/Library/CloudStorage/OneDrive-hydrosolutionsltd/" + "KAZ_DATA/sandro-basins-kaz" +) +REGION = "kaz" +MODELS = ["lstm", "tft", "tide", "tsmixer"] +SHAPEFILE_STEM = "updated_watersheds" +SHAPEFILE_EXTS = [".shp", ".dbf", ".shx", ".prj", ".cpg"] + +# Hive column -> friendly name (only the two series the dashboard needs). +STREAMFLOW_COL = "streamflow" +PRECIP_COL = "total_precipitation_sum" + +QUANTILES = ["Q10", "Q50", "Q90"] +PRED_CSV_COLS = [ + "date", + "Q10_mm_per_day", + "Q50_mm_per_day", + "Q90_mm_per_day", + "Observed_mm_per_day", + "Q10_m3_per_second", + "Q50_m3_per_second", + "Q90_m3_per_second", + "Observed_m3_per_second", +] + + +def mm_per_day_to_m3s(mm_per_day: pd.Series, area_km2: float) -> pd.Series: + """Depth-equivalent runoff (mm/day) -> volumetric discharge (m³/s).""" + return mm_per_day / 1000.0 * (area_km2 * 1e6) / 86400.0 + + +def trim_runs() -> None: + """Copy only the run artifacts the dashboard reads (metrics + metadata). + + Forecasts are not copied as a parquet — the dashboard reads them from the + per-station CSVs in `predictions_csv/`, so the package stays small. + """ + for model in MODELS: + src = RUNS_SRC / model + dst = OUT_DIR / "runs" / model + (dst / "metrics").mkdir(parents=True, exist_ok=True) + for rel in [ + "config.json", + "static_df.csv", + "metrics/test_metrics.csv", + ]: + src_file = src / rel + if not src_file.exists(): + raise FileNotFoundError(f"Expected run artifact missing: {src_file}") + shutil.copy2(src_file, dst / rel) + logger.info("Trimmed run copied: %s", model) + + +def load_predictions(model: str) -> pd.DataFrame: + """Forecast quantiles (mm/day) from the source run.""" + df = pd.read_parquet( + RUNS_SRC / model / "predictions" / "test_predictions.parquet", + columns=["date", "code", "forecast_step", *QUANTILES], + ) + df["code"] = df["code"].astype(int) + df["date"] = pd.to_datetime(df["date"]) + return df + + +def prediction_codes() -> set[int]: + codes: set[int] = set() + for model in MODELS: + codes |= set(load_predictions(model)["code"].unique()) + return codes + + +def build_station_metadata(hive_base: str, codes: set[int]) -> pd.DataFrame: + """stations.csv: gauge_id, code, lat, lon, area_km2, aridity for prediction basins.""" + attrs = pd.read_parquet(_attributes_file(hive_base, REGION)) + attrs = attrs[["gauge_id", "gauge_lat", "gauge_lon", "area", "aridity_ERA5_LAND"]] + attrs["code"] = attrs["gauge_id"].str.removeprefix(f"{REGION}_").astype(int) + stations = ( + attrs[attrs["code"].isin(codes)] + .rename( + columns={ + "gauge_lat": "lat", + "gauge_lon": "lon", + "area": "area_km2", + "aridity_ERA5_LAND": "aridity", + } + ) + .loc[:, ["gauge_id", "code", "lat", "lon", "area_km2", "aridity"]] + .sort_values("code") + .reset_index(drop=True) + ) + missing = codes - set(stations["code"]) + if missing: + logger.warning("No attributes for %d prediction basin(s): %s", len(missing), sorted(missing)) + return stations + + +def load_observed(hive_base: str, code: int) -> pd.DataFrame: + """Observed discharge (mm/day) and precipitation (mm/day) for one basin.""" + parquet = ( + _timeseries_dir(hive_base, REGION) / f"gauge_id={REGION}_{code}" / "data.parquet" + ) + cols = [c for c in ["date", STREAMFLOW_COL, PRECIP_COL] if c] + df = pd.read_parquet(parquet, columns=cols) + df["date"] = pd.to_datetime(df["date"]) + return df.rename( + columns={STREAMFLOW_COL: "discharge_mm_per_day", PRECIP_COL: "P_mm_per_day"} + ).sort_values("date") + + +def write_observations( + obs_by_code: dict[int, pd.DataFrame], date_lo: pd.Timestamp, date_hi: pd.Timestamp +) -> None: + out = OUT_DIR / "data" / "observations" + out.mkdir(parents=True, exist_ok=True) + for code, df in obs_by_code.items(): + window = df[(df["date"] >= date_lo) & (df["date"] <= date_hi)].copy() + window["discharge_mm_per_day"] = window["discharge_mm_per_day"].round(3) + window["P_mm_per_day"] = window["P_mm_per_day"].round(3) + window.to_csv(out / f"{REGION}_{code}.csv", index=False) + logger.info("Wrote %d observation files", len(obs_by_code)) + + +def write_prediction_csvs( + model: str, area_by_code: dict[int, float], obs_by_code: dict[int, pd.DataFrame] +) -> None: + preds = load_predictions(model) + for code, group in preds.groupby("code"): + area = area_by_code.get(int(code)) + obs = obs_by_code.get(int(code)) + if area is None or obs is None: + logger.warning("Skipping %s code %s (no area/observations)", model, code) + continue + observed = obs[["date", "discharge_mm_per_day"]].rename( + columns={"discharge_mm_per_day": "Observed_mm_per_day"} + ) + for lead, lead_df in group.groupby("forecast_step"): + out = pd.DataFrame({"date": lead_df["date"].values}) + for q in QUANTILES: + out[f"{q}_mm_per_day"] = lead_df[q].values + out = out.merge(observed, on="date", how="left") + + # Round mm/day first, then derive m³/s from the rounded values so the + # two unit columns are mutually consistent with the documented formula. + mm_cols = [f"{q}_mm_per_day" for q in QUANTILES] + ["Observed_mm_per_day"] + out[mm_cols] = out[mm_cols].round(2) + for q in QUANTILES: + out[f"{q}_m3_per_second"] = mm_per_day_to_m3s( + out[f"{q}_mm_per_day"], area + ).round(2) + out["Observed_m3_per_second"] = mm_per_day_to_m3s( + out["Observed_mm_per_day"], area + ).round(2) + out = out[PRED_CSV_COLS].sort_values("date") + + lead_dir = OUT_DIR / "predictions_csv" / model / f"lead_{int(lead):02d}" + lead_dir.mkdir(parents=True, exist_ok=True) + out.to_csv(lead_dir / f"{REGION}_{int(code)}.csv", index=False) + logger.info("Wrote prediction CSVs for model: %s", model) + + +def copy_shapefile(hive_base: str) -> None: + dst = OUT_DIR / "data" + dst.mkdir(parents=True, exist_ok=True) + copied = 0 + for ext in SHAPEFILE_EXTS: + src = Path(hive_base) / f"{SHAPEFILE_STEM}{ext}" + if src.exists(): + shutil.copy2(src, dst / src.name) + copied += 1 + if copied == 0: + raise FileNotFoundError(f"No shapefile parts found at {hive_base}/{SHAPEFILE_STEM}.*") + logger.info("Copied %d shapefile part(s)", copied) + + +def copy_template_files() -> None: + for name in ["kaz_dashboard.py", "requirements.txt", "README.md"]: + src = TEMPLATE_DIR / name + if not src.exists(): + raise FileNotFoundError(f"Missing template file: {src}") + shutil.copy2(src, OUT_DIR / name) + logger.info("Copied dashboard, requirements, README") + + +def build(hive_base: str) -> None: + if not Path(hive_base).exists(): + raise FileNotFoundError(f"Hive base path not found: {hive_base}") + if OUT_DIR.exists(): + shutil.rmtree(OUT_DIR) + OUT_DIR.mkdir(parents=True) + + trim_runs() + codes = {int(c) for c in prediction_codes()} + logger.info("Prediction basins: %d", len(codes)) + + stations = build_station_metadata(hive_base, codes) + (OUT_DIR / "data").mkdir(parents=True, exist_ok=True) + stations.to_csv(OUT_DIR / "data" / "stations.csv", index=False) + area_by_code = dict(zip(stations["code"], stations["area_km2"], strict=True)) + + obs_by_code = {code: load_observed(hive_base, code) for code in sorted(codes)} + + # Observation window = prediction span (± 60 days margin for the plot). + all_dates = pd.concat([load_predictions(m)["date"] for m in MODELS]) + date_lo = all_dates.min() - pd.Timedelta(days=60) + date_hi = all_dates.max() + pd.Timedelta(days=60) + write_observations(obs_by_code, date_lo, date_hi) + + for model in MODELS: + write_prediction_csvs(model, area_by_code, obs_by_code) + + copy_shapefile(hive_base) + copy_template_files() + logger.info("Done. Package assembled at: %s", OUT_DIR) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build the KAZ dashboard export package.") + parser.add_argument( + "--hive-base", + default=DEFAULT_HIVE_BASE, + help="Path to the Hive basin dataset root. Default: %(default)s", + ) + args = parser.parse_args() + build(args.hive_base) + + +if __name__ == "__main__": + main() From 3e805e789a030af1a62bc5ed95cb528e16b43b23 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Wed, 24 Jun 2026 10:01:10 +0200 Subject: [PATCH 16/20] Create append_kaz_discharge_level.py --- scripts/append_kaz_discharge_level.py | 209 ++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 scripts/append_kaz_discharge_level.py diff --git a/scripts/append_kaz_discharge_level.py b/scripts/append_kaz_discharge_level.py new file mode 100644 index 0000000..a49e47c --- /dev/null +++ b/scripts/append_kaz_discharge_level.py @@ -0,0 +1,209 @@ +"""Append Bauyrzhan KAZ discharge & water-level (RDS) into the hive dataset. + +Discharge (m3/s) is converted to mm/day (column ``streamflow``, replacing any +existing values) and also kept raw (``discharge_m3s``). Water level (cm) and the +ice-regime flag are added as ``water_level_cm`` and ``swo``. Stations absent from +the hive (no basin area) get new timeseries-only partitions with ``streamflow`` +left NaN. Rows outside the forcing range are kept (forcing columns stay NaN). + +``pyreadr`` cannot parse these RDS files, so they are exported to CSV via R +(``data.table::fwrite``) as an intermediate step. +""" + +import argparse +import logging +import shutil +import subprocess +from datetime import datetime +from pathlib import Path + +import numpy as np +import pandas as pd + +from st_forecast.data_utils.transformers import m3_to_mm + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger("append_kaz") + +DEFAULT_RDS_DIR = Path( + "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/" + "SAPPHIRE_Central_Asia_Technical_Work/data/KAZ_DATA/" + "bauyrzhan-complete-KAZ-database" +) +DEFAULT_HIVE = Path( + "/Users/sandrohunziker/Library/CloudStorage/OneDrive-hydrosolutionsltd/" + "KAZ_DATA/sandro-basins-kaz" +) +DEFAULT_SCRATCH = Path( + "/private/tmp/claude-501/" + "-Users-sandrohunziker-hydrosolutions-Dropbox-Sandro-Hunziker-SAPPHIRE-Central-Asia-Technical-Work-code-machine-learning-hydrology-DL-Short-Term-Forecasting/" + "0c54453f-d60f-4a50-8d38-7eb8ee46e647/scratchpad" +) +REGION = "kaz" + + +def timeseries_root(hive: Path) -> Path: + return ( + hive + / f"REGION_NAME={REGION}" + / "data_type=timeseries" + / "temporal_resolution=daily" + ) + + +def export_rds_to_csv(rds_path: Path, out_csv: Path) -> None: + """Export an RDS data.frame to UTF-8 CSV via R's data.table::fwrite.""" + r_code = ( + "args <- commandArgs(TRUE); " + "suppressMessages(library(data.table)); " + "x <- as.data.table(readRDS(args[1])); " + "fwrite(x, args[2])" + ) + logger.info("Exporting %s -> %s via Rscript", rds_path.name, out_csv.name) + subprocess.run( + ["Rscript", "-e", r_code, str(rds_path), str(out_csv)], + check=True, + capture_output=True, + text=True, + ) + + +def parse_numeric(series: pd.Series) -> pd.Series: + """Coerce text values to float: comma-decimal -> dot, junk -> NaN.""" + cleaned = series.astype(str).str.strip().str.replace(",", ".", regex=False) + return pd.to_numeric(cleaned, errors="coerce") + + +def read_obs_csv(csv_path: Path, value_name: str, has_swo: bool) -> pd.DataFrame: + """Read an exported obs CSV into normalized long format.""" + usecols = ["site", "sdate", "value"] + (["swo"] if has_swo else []) + dtypes = {c: str for c in usecols} + df = pd.read_csv( + csv_path, + usecols=usecols, + dtype=dtypes, + na_filter=False, + encoding="utf-8", + encoding_errors="replace", + ) + out = pd.DataFrame( + { + "site": df["site"].str.strip(), + "date": df["sdate"].str.strip(), + value_name: parse_numeric(df["value"]), + } + ) + if has_swo: + out["swo"] = df["swo"] # ice-regime flag kept verbatim (string) + + n_fail = int((parse_numeric(df["value"]).isna() & (df["value"].str.strip() != "")).sum()) + before = len(out) + out = out.drop_duplicates(subset=["site", "date"], keep="first") + logger.info( + "%s: %d rows, %d unparseable value tokens, %d duplicate (site,date) dropped", + csv_path.name, + before, + n_fail, + before - len(out), + ) + return out + + +def backup_hive(ts_root: Path, scratch: Path) -> Path: + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_dir = scratch / f"hive_backup_{stamp}" + files = sorted(ts_root.glob("gauge_id=*/data.parquet")) + if not files: + raise FileNotFoundError(f"No existing parquet files found under {ts_root}") + for src in files: + dst = backup_dir / src.parent.name / src.name + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + logger.info("Backed up %d existing parquet files to %s", len(files), backup_dir) + return backup_dir + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rds-dir", type=Path, default=DEFAULT_RDS_DIR) + parser.add_argument("--hive", type=Path, default=DEFAULT_HIVE) + parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH) + args = parser.parse_args() + + ts_root = timeseries_root(args.hive) + args.scratch.mkdir(parents=True, exist_ok=True) + + # Step 0: backup + backup_hive(ts_root, args.scratch) + + # Step 1: RDS -> CSV + dis_csv = args.scratch / "all_dis.csv" + lev_csv = args.scratch / "all_lev.csv" + export_rds_to_csv(args.rds_dir / "all_dis.RDS", dis_csv) + export_rds_to_csv(args.rds_dir / "all_lev.RDS", lev_csv) + + # Step 2: normalize + dis = read_obs_csv(dis_csv, "discharge_m3s", has_swo=False) + lev = read_obs_csv(lev_csv, "water_level_cm", has_swo=True) + dis_groups = dict(tuple(dis.groupby("site"))) + lev_groups = dict(tuple(lev.groupby("site"))) + + # areas keyed by bare site code + attrs = pd.read_parquet( + args.hive / f"REGION_NAME={REGION}" / "data_type=attributes" / "data.parquet" + ) + area_by_site = { + gid.replace(f"{REGION}_", ""): area + for gid, area in zip(attrs["gauge_id"], attrs["area"]) + } + hive_sites = set(area_by_site) + all_sites = set(dis_groups) | set(lev_groups) + new_sites = sorted(all_sites - hive_sites) + + def build_obs(site: str) -> pd.DataFrame: + d = dis_groups.get(site) + ll = lev_groups.get(site) + if d is not None and ll is not None: + obs = pd.merge(d.drop(columns="site"), ll.drop(columns="site"), on="date", how="outer") + elif d is not None: + obs = d.drop(columns="site").copy() + obs["water_level_cm"] = np.nan + obs["swo"] = "" + else: + obs = ll.drop(columns="site").copy() + obs["discharge_m3s"] = np.nan + return obs[["date", "discharge_m3s", "water_level_cm", "swo"]] + + # Step 3: update existing hive gauges + rows_beyond = 0 + updated = 0 + for site in sorted(hive_sites): + path = ts_root / f"gauge_id={REGION}_{site}" / "data.parquet" + if not path.exists(): + logger.warning("Hive gauge %s has no parquet, skipping", site) + continue + existing = pd.read_parquet(path) + obs = build_obs(site) + n_existing_dates = existing["date"].nunique() + merged = existing.merge(obs, on="date", how="outer").sort_values("date") + merged["streamflow"] = m3_to_mm(merged["discharge_m3s"], area_by_site[site]) + merged.to_parquet(path, index=False) + rows_beyond += merged["date"].nunique() - n_existing_dates + updated += 1 + logger.info("Updated %d hive gauges (+%d new dates beyond forcing range)", updated, rows_beyond) + + # Step 4: new timeseries-only partitions + for site in new_sites: + obs = build_obs(site).sort_values("date") + obs["streamflow"] = np.nan # no area -> cannot convert + path = ts_root / f"gauge_id={REGION}_{site}" / "data.parquet" + path.parent.mkdir(parents=True, exist_ok=True) + obs[["date", "discharge_m3s", "streamflow", "water_level_cm", "swo"]].to_parquet( + path, index=False + ) + logger.info("Created %d new timeseries-only partitions", len(new_sites)) + logger.info("Done. Hive sites: %d | new sites: %d", len(hive_sites), len(new_sites)) + + +if __name__ == "__main__": + main() From 7006a35053a4cae3850d2fbfcdf60258ba58ac5f Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:45:22 +0200 Subject: [PATCH 17/20] kaz full integration - non int codes are exluded --- scripts/replace_kaz_forcing.py | 202 +++++++++++++++++++++++++ st_forecast/data_utils/hive_loaders.py | 74 ++++++--- tests/test_hive_loaders.py | 89 +++++++++++ 3 files changed, 348 insertions(+), 17 deletions(-) create mode 100644 scripts/replace_kaz_forcing.py create mode 100644 tests/test_hive_loaders.py diff --git a/scripts/replace_kaz_forcing.py b/scripts/replace_kaz_forcing.py new file mode 100644 index 0000000..205ed4b --- /dev/null +++ b/scripts/replace_kaz_forcing.py @@ -0,0 +1,202 @@ +"""Replace KAZ forcing & static with the new caravanify ERA5 dataset. + +The new dataset (``final_hdx/``) carries a different ERA5 product (12 daily vars, +1980-2025) and minimal static (area/lat/lon) for 373 basins. This script: + +- drops the old 43 ERA5-LAND forcing columns and joins the 12 new ERA5 vars, +- gives every hive partition a uniform schema (basins without new ERA5 get NaN + forcing, keeping their discharge), +- adopts the new ``drainage_area`` and recomputes ``streamflow`` (mm/day), +- keeps the rich 212-col attributes table, adding rows for the new basins. + +Backs up every timeseries parquet + the attributes parquet before overwriting. +""" + +import argparse +import logging +import shutil +from datetime import datetime +from pathlib import Path + +import numpy as np +import pandas as pd + +from st_forecast.data_utils.transformers import m3_to_mm + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger("replace_kaz_forcing") + +DEFAULT_NEW = Path( + "/Users/sandrohunziker/hydrosolutions Dropbox/Sandro Hunziker/" + "SAPPHIRE_Central_Asia_Technical_Work/data/KAZ_DATA/full/" + "caravanify-kaz-hdx/final_hdx" +) +DEFAULT_HIVE = Path( + "/Users/sandrohunziker/Library/CloudStorage/OneDrive-hydrosolutionsltd/" + "KAZ_DATA/sandro-basins-kaz" +) +DEFAULT_SCRATCH = Path( + "/private/tmp/claude-501/" + "-Users-sandrohunziker-hydrosolutions-Dropbox-Sandro-Hunziker-SAPPHIRE-Central-Asia-Technical-Work-code-machine-learning-hydrology-DL-Short-Term-Forecasting/" + "0c54453f-d60f-4a50-8d38-7eb8ee46e647/scratchpad" +) +REGION = "kaz" + +# New ERA5 forcing variables, in canonical column order. +ERA5_VARS = [ + "total_precipitation_sum", + "temperature_2m_mean", + "temperature_2m_min", + "temperature_2m_max", + "dewpoint_temperature_2m_mean", + "surface_net_thermal_radiation_sum", + "surface_net_solar_radiation_sum", + "u_component_of_wind_10m_mean", + "v_component_of_wind_10m_mean", + "snow_depth_water_equivalent_mean", + "surface_pressure_mean", + "potential_evaporation_sum", +] +OBS_COLS = ["streamflow", "discharge_m3s", "water_level_cm", "swo"] +CANONICAL = ["date"] + ERA5_VARS + OBS_COLS + + +def timeseries_root(hive: Path) -> Path: + return ( + hive + / f"REGION_NAME={REGION}" + / "data_type=timeseries" + / "temporal_resolution=daily" + ) + + +def attributes_path(hive: Path) -> Path: + return hive / f"REGION_NAME={REGION}" / "data_type=attributes" / "data.parquet" + + +def backup(ts_root: Path, attr_path: Path, scratch: Path) -> Path: + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_dir = scratch / f"forcing_backup_{stamp}" + files = sorted(ts_root.glob("gauge_id=*/data.parquet")) + if not files: + raise FileNotFoundError(f"No timeseries parquet files under {ts_root}") + for src in files: + dst = backup_dir / "timeseries" / src.parent.name / src.name + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + if attr_path.exists(): + shutil.copy2(attr_path, backup_dir / "attributes.parquet") + logger.info("Backed up %d timeseries + attributes to %s", len(files), backup_dir) + return backup_dir + + +def load_new_era5(dyn_path: Path) -> pd.DataFrame: + """Read one basin's scalar_dynamic, return ERA5 vars keyed by date string.""" + dyn = pd.read_parquet(dyn_path) + dyn["date"] = pd.to_datetime(dyn["time"]).dt.strftime("%Y-%m-%d") + return dyn[["date"] + ERA5_VARS] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--new", type=Path, default=DEFAULT_NEW) + parser.add_argument("--hive", type=Path, default=DEFAULT_HIVE) + parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH) + args = parser.parse_args() + + ts_root = timeseries_root(args.hive) + attr_path = attributes_path(args.hive) + args.scratch.mkdir(parents=True, exist_ok=True) + + # Step 0: backup + backup(ts_root, attr_path, args.scratch) + + # Step 1: new static + ERA5 index + static = pd.read_parquet(args.new / "scalar_static.parquet") + new_area = dict(zip(static["basin_id"], static["drainage_area"])) + dyn_by_basin = { + p.parent.name.replace("basin=", ""): p + for p in args.new.glob("basin=*/scalar_dynamic.parquet") + } + logger.info( + "New static: %d basins | dynamic ERA5: %d basins", len(new_area), len(dyn_by_basin) + ) + + # Step 2: rewrite every timeseries partition + ts_dirs = sorted(ts_root.glob("gauge_id=*")) + forced = 0 + discharge_only = 0 + area_changed = 0 + for d in ts_dirs: + gid = d.name.replace("gauge_id=", "") + path = d / "data.parquet" + existing = pd.read_parquet(path) + kept = existing[[c for c in OBS_COLS if c in existing.columns] + ["date"]].copy() + + dyn_path = dyn_by_basin.get(gid) + if dyn_path is not None: + era5 = load_new_era5(dyn_path) + merged = kept.merge(era5, on="date", how="outer") + forced += 1 + else: + merged = kept.copy() + for v in ERA5_VARS: + merged[v] = np.nan + discharge_only += 1 + + # recompute streamflow with the new area where available + if gid in new_area and "discharge_m3s" in merged.columns: + merged["streamflow"] = m3_to_mm(merged["discharge_m3s"], new_area[gid]) + area_changed += 1 + # ensure all obs cols exist + for c in OBS_COLS: + if c not in merged.columns: + merged[c] = np.nan if c != "swo" else "" + + out = merged.reindex(columns=CANONICAL).sort_values("date") + out.to_parquet(path, index=False) + + logger.info( + "Rewrote %d partitions: %d with ERA5, %d discharge-only, %d streamflow-recomputed", + len(ts_dirs), + forced, + discharge_only, + area_changed, + ) + + # Step 3: attributes + attr = pd.read_parquet(attr_path) + attr_ids = set(attr["gauge_id"]) + # update area for overlap basins + overlap_mask = attr["gauge_id"].isin(new_area) + attr.loc[overlap_mask, "area"] = attr.loc[overlap_mask, "gauge_id"].map(new_area) + n_overlap = int(overlap_mask.sum()) + + # append rows for new basins missing from attributes + new_rows = [] + for bid, area in new_area.items(): + if bid in attr_ids: + continue + row = static[static["basin_id"] == bid].iloc[0] + new_rows.append( + { + "gauge_id": bid, + "area": area, + "gauge_lat": row["outlet_lat"], + "gauge_lon": row["outlet_lon"], + } + ) + if new_rows: + attr = pd.concat([attr, pd.DataFrame(new_rows)], ignore_index=True) + attr.to_parquet(attr_path, index=False) + logger.info( + "Attributes: updated area for %d overlap basins, added %d new rows -> %d total", + n_overlap, + len(new_rows), + len(attr), + ) + logger.info("Done.") + + +if __name__ == "__main__": + main() diff --git a/st_forecast/data_utils/hive_loaders.py b/st_forecast/data_utils/hive_loaders.py index 86fbc57..93f1cdd 100644 --- a/st_forecast/data_utils/hive_loaders.py +++ b/st_forecast/data_utils/hive_loaders.py @@ -36,7 +36,11 @@ def _basin_id_from_path(parquet_path: Path, region: str) -> str: raise ValueError(f"Unexpected partition folder: {parent}") gauge_id = parent[len(prefix) :] region_prefix = f"{region}_" - return gauge_id[len(region_prefix) :] if gauge_id.startswith(region_prefix) else gauge_id + return ( + gauge_id[len(region_prefix) :] + if gauge_id.startswith(region_prefix) + else gauge_id + ) def _to_int_or_str(value: str) -> int | str: @@ -46,6 +50,34 @@ def _to_int_or_str(value: str) -> int | str: return value +def _is_integer_code(value: str) -> bool: + """True if the gauge code can be cast to int (the only form supported now).""" + try: + int(value) + except (ValueError, TypeError): + return False + return True + + +def _warn_excluded_noninteger(codes: list[str], source: str) -> None: + """Emit a single warning that non-integer gauge codes are unsupported.""" + if not codes: + return + shown = sorted(codes) + preview = shown[:10] + suffix = "..." if len(shown) > 10 else "" + logger.warning( + "Excluding %d non-integer gauge code(s) from %s: %s%s. " + "Non-integer/suffixed basin codes (e.g. '19073.1', '11147_1') are not " + "supported by the training pipeline at this time and are skipped; the " + "underlying data is left untouched on disk.", + len(shown), + source, + preview, + suffix, + ) + + def discover_basin_files( base_path: str, region: str, @@ -59,11 +91,20 @@ def discover_basin_files( if not all_files: raise FileNotFoundError(f"No parquet files found under {ts_dir}") + # Only integer-coded gauges are supported; skip suffixed sub-gauges with a warning. + excluded = [ + _basin_id_from_path(f, region) + for f in all_files + if not _is_integer_code(_basin_id_from_path(f, region)) + ] + _warn_excluded_noninteger(excluded, "timeseries discovery") + all_files = [ + f for f in all_files if _is_integer_code(_basin_id_from_path(f, region)) + ] + if basin_ids is not None: wanted = {str(b) for b in basin_ids} - selected = [ - f for f in all_files if _basin_id_from_path(f, region) in wanted - ] + selected = [f for f in all_files if _basin_id_from_path(f, region) in wanted] missing = wanted - {_basin_id_from_path(f, region) for f in selected} for m in missing: logger.warning(f"Basin not found in hive layout: {region}_{m}") @@ -92,7 +133,9 @@ def load_single_basin_timeseries( if end_date is not None: df = df[df["date"] <= pd.Timestamp(end_date)] - keep_cols = ["date", "code"] + [v for v in variable_mapping.values() if v in df.columns] + keep_cols = ["date", "code"] + [ + v for v in variable_mapping.values() if v in df.columns + ] return df[keep_cols].copy() @@ -162,22 +205,19 @@ def load_hive_attributes( if "gauge_id" in df.columns: region_prefix = f"{region}_" code_strs = df["gauge_id"].apply( - lambda g: g[len(region_prefix) :] if isinstance(g, str) and g.startswith(region_prefix) else g + lambda g: g[len(region_prefix) :] + if isinstance(g, str) and g.startswith(region_prefix) + else g ) - try: - df["code"] = code_strs.astype(int) - except (ValueError, TypeError): - df["code"] = code_strs + # Drop non-integer-coded rows so the cast (and downstream merges) stay int-typed. + is_int = code_strs.apply(_is_integer_code) + _warn_excluded_noninteger(code_strs[~is_int].astype(str).tolist(), "attributes") + df = df[is_int.to_numpy()].copy() + df["code"] = code_strs[is_int].astype(int) df["CODE"] = df["code"] if basin_ids is not None: - if df["code"].dtype in ("int64", "int32", "float64"): - try: - basin_ids_typed = [int(b) for b in basin_ids] - except (ValueError, TypeError): - basin_ids_typed = list(basin_ids) - else: - basin_ids_typed = list(basin_ids) + basin_ids_typed = [int(b) for b in basin_ids if _is_integer_code(b)] df = df[df["code"].isin(basin_ids_typed)] df.index = df["CODE"] diff --git a/tests/test_hive_loaders.py b/tests/test_hive_loaders.py new file mode 100644 index 0000000..127e560 --- /dev/null +++ b/tests/test_hive_loaders.py @@ -0,0 +1,89 @@ +"""Tests for hive loaders: non-integer gauge codes are excluded with a warning.""" + +import logging + +import pandas as pd + +from st_forecast.data_utils.hive_loaders import ( + discover_basin_files, + load_hive_attributes, +) + +REGION = "kaz" +INT_CODES = ["11001", "11002"] +SUFFIXED_CODE = "19073.1" + + +def _build_hive(root, codes: list[str]) -> None: + ts_dir = ( + root + / f"REGION_NAME={REGION}" + / "data_type=timeseries" + / "temporal_resolution=daily" + ) + for code in codes: + d = ts_dir / f"gauge_id={REGION}_{code}" + d.mkdir(parents=True) + pd.DataFrame( + {"date": ["2020-01-01"], "discharge": [1.0], "streamflow": [0.1]} + ).to_parquet(d / "data.parquet", index=False) + + attr_dir = root / f"REGION_NAME={REGION}" / "data_type=attributes" + attr_dir.mkdir(parents=True) + pd.DataFrame( + { + "gauge_id": [f"{REGION}_{c}" for c in codes], + "area": [100.0 * (i + 1) for i in range(len(codes))], + "p_mean": [1.0] * len(codes), + } + ).to_parquet(attr_dir / "data.parquet", index=False) + + +class TestDiscoverBasinFiles: + def test_excludes_noninteger_codes_and_warns(self, tmp_path, caplog): + _build_hive(tmp_path, INT_CODES + [SUFFIXED_CODE]) + + with caplog.at_level(logging.WARNING): + files = discover_basin_files(str(tmp_path), REGION) + + found = {f.parent.name for f in files} + assert found == {f"gauge_id={REGION}_{c}" for c in INT_CODES} + assert f"gauge_id={REGION}_{SUFFIXED_CODE}" not in found + assert any( + "non-integer" in r.message.lower() and SUFFIXED_CODE in r.message + for r in caplog.records + ) + + def test_no_warning_when_all_integer(self, tmp_path, caplog): + _build_hive(tmp_path, INT_CODES) + + with caplog.at_level(logging.WARNING): + files = discover_basin_files(str(tmp_path), REGION) + + assert len(files) == len(INT_CODES) + assert not any("non-integer" in r.message.lower() for r in caplog.records) + + +class TestLoadHiveAttributes: + def test_drops_noninteger_rows_with_int_dtype_and_warns(self, tmp_path, caplog): + _build_hive(tmp_path, INT_CODES + [SUFFIXED_CODE]) + + with caplog.at_level(logging.WARNING): + attrs = load_hive_attributes(str(tmp_path), REGION) + + assert len(attrs) == len(INT_CODES) + assert pd.api.types.is_integer_dtype(attrs.index.dtype) + assert set(attrs.index) == {int(c) for c in INT_CODES} + assert any( + "non-integer" in r.message.lower() and SUFFIXED_CODE in r.message + for r in caplog.records + ) + + def test_basin_ids_filter_ignores_noninteger_request(self, tmp_path): + _build_hive(tmp_path, INT_CODES + [SUFFIXED_CODE]) + + attrs = load_hive_attributes( + str(tmp_path), REGION, basin_ids=["11001", SUFFIXED_CODE] + ) + + assert set(attrs.index) == {11001} From face22b3dcea9f26674837046f50c11d5eb780b6 Mon Sep 17 00:00:00 2001 From: sandrohuni Date: Tue, 30 Jun 2026 15:05:24 +0200 Subject: [PATCH 18/20] fix: account for training_length in series min-length filter RNN models require series length >= training_length + 1, but the filter only checked input_chunk_length + forecast_horizon + 1, letting too-short series through and crashing darts. Thread training_length into _build_darts_timeseries and use max(input+horizon, training_length)+1. Co-Authored-By: Claude Opus 4.8 (1M context) --- .bumpversion.toml | 2 +- pyproject.toml | 2 +- st_forecast/__init__.py | 2 +- st_forecast/data_utils/preparation.py | 6 +++++- st_forecast/pipeline/artifacts.py | 1 + uv.lock | 8 ++------ 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 5168549..c7a23f9 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.1.9" +current_version = "0.1.10" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" diff --git a/pyproject.toml b/pyproject.toml index 7184aab..1bd3fdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "st-forecast" -version = "0.1.9" +version = "0.1.10" description = "Deep-learning short-term discharge forecasting for multi-basin hydrological systems; entry point to the SAPPHIRE forecast tools." readme = "README.md" requires-python = ">=3.11" diff --git a/st_forecast/__init__.py b/st_forecast/__init__.py index f0a8fa9..80191c9 100644 --- a/st_forecast/__init__.py +++ b/st_forecast/__init__.py @@ -13,7 +13,7 @@ from st_forecast.pipeline.inference import DEFAULT_QUANTILES from st_forecast.pipeline.workflow import TrainModelResult, train_model -__version__ = "0.1.9" +__version__ = "0.1.10" # Package-level constants BASIN_ID_COL = "code" diff --git a/st_forecast/data_utils/preparation.py b/st_forecast/data_utils/preparation.py index 6cad60d..cd82254 100644 --- a/st_forecast/data_utils/preparation.py +++ b/st_forecast/data_utils/preparation.py @@ -548,6 +548,7 @@ def _build_timeseries_pipeline( adjusted_features, input_chunk_length, forecast_horizon, + training_length=config.get("training_length", 0), date_col=date_col, basin_id_col=basin_id_col, target_col=target_col, @@ -629,6 +630,7 @@ def _build_darts_timeseries( features: list[str], input_chunk_length: int, forecast_horizon: int, + training_length: int = 0, date_col: str = "date", basin_id_col: str = "code", target_col: str = "discharge", @@ -678,7 +680,9 @@ def _build_darts_timeseries( # Extract subseries to handle NaN gaps in ANY column (discharge or features) # This matches the original behavior in get_data.py - min_length = input_chunk_length + forecast_horizon + 1 + # RNN models need series >= training_length + 1; block models need + # input_chunk_length + forecast_horizon + 1. Take whichever is larger. + min_length = max(input_chunk_length + forecast_horizon, training_length) + 1 final_ts = [] for ts in result_ts: sliced_timeseries = extract_subseries(ts, min_gap_size=1, mode="any") diff --git a/st_forecast/pipeline/artifacts.py b/st_forecast/pipeline/artifacts.py index 0d06a78..a4b5e2a 100644 --- a/st_forecast/pipeline/artifacts.py +++ b/st_forecast/pipeline/artifacts.py @@ -206,6 +206,7 @@ def to_prepare_config(self) -> dict: "shift_moving_avg": self.shift_moving_avg, "forecast_horizon": self.forecast_horizon, "input_chunk_length": self.input_chunk_length, + "training_length": self.extra_hyperparams.get("training_length", 0), "scale_discharge": self.scale_discharge_bool, "discharge_scaler_type": self.discharge_scaler_type, "global_scaling": self.global_scaling, diff --git a/uv.lock b/uv.lock index f0552fb..3efa44f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.12'", @@ -781,7 +781,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, { url = "https://files.pythonhosted.org/packages/ae/8f/95d48d7e3d433e6dae5b1682e4292242a53f22df82e6d3dda81b1701a960/greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3", size = 644646, upload-time = "2025-08-07T13:45:26.523Z" }, - { url = "https://files.pythonhosted.org/packages/d5/5e/405965351aef8c76b8ef7ad370e5da58d57ef6068df197548b015464001a/greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633", size = 640519, upload-time = "2025-08-07T13:53:13.928Z" }, { url = "https://files.pythonhosted.org/packages/25/5d/382753b52006ce0218297ec1b628e048c4e64b155379331f25a7316eb749/greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079", size = 639707, upload-time = "2025-08-07T13:18:27.146Z" }, { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, @@ -792,7 +791,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, @@ -803,7 +801,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, { url = "https://files.pythonhosted.org/packages/f7/0b/bc13f787394920b23073ca3b6c4a7a21396301ed75a655bcb47196b50e6e/greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc", size = 655191, upload-time = "2025-08-07T13:45:29.752Z" }, - { url = "https://files.pythonhosted.org/packages/f2/d6/6adde57d1345a8d0f14d31e4ab9c23cfe8e2cd39c3baf7674b4b0338d266/greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a", size = 649516, upload-time = "2025-08-07T13:53:16.314Z" }, { url = "https://files.pythonhosted.org/packages/7f/3b/3a3328a788d4a473889a2d403199932be55b1b0060f4ddd96ee7cdfcad10/greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504", size = 652169, upload-time = "2025-08-07T13:18:32.861Z" }, { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, @@ -814,7 +811,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, { url = "https://files.pythonhosted.org/packages/c0/aa/687d6b12ffb505a4447567d1f3abea23bd20e73a5bed63871178e0831b7a/greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5", size = 699218, upload-time = "2025-08-07T13:45:30.969Z" }, - { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, { url = "https://files.pythonhosted.org/packages/23/6e/74407aed965a4ab6ddd93a7ded3180b730d281c77b765788419484cdfeef/greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269", size = 1612508, upload-time = "2025-11-04T12:42:23.427Z" }, @@ -3155,7 +3151,7 @@ wheels = [ [[package]] name = "st-forecast" -version = "0.1.9" +version = "0.1.10" source = { editable = "." } dependencies = [ { name = "click" }, From 4644f10c270553f212462d1c530c52093ee0a6ae Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:42:56 +0200 Subject: [PATCH 19/20] ruff formatiing --- examples/lamah_workflow.py | 2 +- scripts/append_kaz_discharge_level.py | 20 ++++++--- scripts/build_kaz_export.py | 18 ++++++-- scripts/explore_kaz.py | 4 +- scripts/replace_kaz_forcing.py | 8 +++- st_forecast/operational/sapphire_predictor.py | 17 ++++---- st_forecast/pipeline/artifacts.py | 4 +- tests/test_public_api_smoke.py | 42 +++++++++++-------- 8 files changed, 68 insertions(+), 47 deletions(-) diff --git a/examples/lamah_workflow.py b/examples/lamah_workflow.py index 2d1e3a5..534c569 100644 --- a/examples/lamah_workflow.py +++ b/examples/lamah_workflow.py @@ -142,7 +142,7 @@ def create_config(work_dir: Path) -> RunConfig: """Create RunConfig for LamaH example basins.""" return RunConfig( region="lamah_ce", - model_type="RNN", # Defaults to LSTM when using RNN + model_type="RNN", # Defaults to LSTM when using RNN model_name="RNN_lamah_example", input_chunk_length=90, forecast_horizon=3, diff --git a/scripts/append_kaz_discharge_level.py b/scripts/append_kaz_discharge_level.py index a49e47c..6849b80 100644 --- a/scripts/append_kaz_discharge_level.py +++ b/scripts/append_kaz_discharge_level.py @@ -96,7 +96,9 @@ def read_obs_csv(csv_path: Path, value_name: str, has_swo: bool) -> pd.DataFrame if has_swo: out["swo"] = df["swo"] # ice-regime flag kept verbatim (string) - n_fail = int((parse_numeric(df["value"]).isna() & (df["value"].str.strip() != "")).sum()) + n_fail = int( + (parse_numeric(df["value"]).isna() & (df["value"].str.strip() != "")).sum() + ) before = len(out) out = out.drop_duplicates(subset=["site", "date"], keep="first") logger.info( @@ -164,7 +166,9 @@ def build_obs(site: str) -> pd.DataFrame: d = dis_groups.get(site) ll = lev_groups.get(site) if d is not None and ll is not None: - obs = pd.merge(d.drop(columns="site"), ll.drop(columns="site"), on="date", how="outer") + obs = pd.merge( + d.drop(columns="site"), ll.drop(columns="site"), on="date", how="outer" + ) elif d is not None: obs = d.drop(columns="site").copy() obs["water_level_cm"] = np.nan @@ -190,7 +194,11 @@ def build_obs(site: str) -> pd.DataFrame: merged.to_parquet(path, index=False) rows_beyond += merged["date"].nunique() - n_existing_dates updated += 1 - logger.info("Updated %d hive gauges (+%d new dates beyond forcing range)", updated, rows_beyond) + logger.info( + "Updated %d hive gauges (+%d new dates beyond forcing range)", + updated, + rows_beyond, + ) # Step 4: new timeseries-only partitions for site in new_sites: @@ -198,9 +206,9 @@ def build_obs(site: str) -> pd.DataFrame: obs["streamflow"] = np.nan # no area -> cannot convert path = ts_root / f"gauge_id={REGION}_{site}" / "data.parquet" path.parent.mkdir(parents=True, exist_ok=True) - obs[["date", "discharge_m3s", "streamflow", "water_level_cm", "swo"]].to_parquet( - path, index=False - ) + obs[ + ["date", "discharge_m3s", "streamflow", "water_level_cm", "swo"] + ].to_parquet(path, index=False) logger.info("Created %d new timeseries-only partitions", len(new_sites)) logger.info("Done. Hive sites: %d | new sites: %d", len(hive_sites), len(new_sites)) diff --git a/scripts/build_kaz_export.py b/scripts/build_kaz_export.py index 0928c7d..4f72a59 100644 --- a/scripts/build_kaz_export.py +++ b/scripts/build_kaz_export.py @@ -130,14 +130,20 @@ def build_station_metadata(hive_base: str, codes: set[int]) -> pd.DataFrame: ) missing = codes - set(stations["code"]) if missing: - logger.warning("No attributes for %d prediction basin(s): %s", len(missing), sorted(missing)) + logger.warning( + "No attributes for %d prediction basin(s): %s", + len(missing), + sorted(missing), + ) return stations def load_observed(hive_base: str, code: int) -> pd.DataFrame: """Observed discharge (mm/day) and precipitation (mm/day) for one basin.""" parquet = ( - _timeseries_dir(hive_base, REGION) / f"gauge_id={REGION}_{code}" / "data.parquet" + _timeseries_dir(hive_base, REGION) + / f"gauge_id={REGION}_{code}" + / "data.parquet" ) cols = [c for c in ["date", STREAMFLOW_COL, PRECIP_COL] if c] df = pd.read_parquet(parquet, columns=cols) @@ -208,7 +214,9 @@ def copy_shapefile(hive_base: str) -> None: shutil.copy2(src, dst / src.name) copied += 1 if copied == 0: - raise FileNotFoundError(f"No shapefile parts found at {hive_base}/{SHAPEFILE_STEM}.*") + raise FileNotFoundError( + f"No shapefile parts found at {hive_base}/{SHAPEFILE_STEM}.*" + ) logger.info("Copied %d shapefile part(s)", copied) @@ -254,7 +262,9 @@ def build(hive_base: str) -> None: def main() -> None: - parser = argparse.ArgumentParser(description="Build the KAZ dashboard export package.") + parser = argparse.ArgumentParser( + description="Build the KAZ dashboard export package." + ) parser.add_argument( "--hive-base", default=DEFAULT_HIVE_BASE, diff --git a/scripts/explore_kaz.py b/scripts/explore_kaz.py index 26967a5..6ccad73 100644 --- a/scripts/explore_kaz.py +++ b/scripts/explore_kaz.py @@ -1181,9 +1181,7 @@ def main() -> None: returned_objects=["last_object_clicked"], key="kaz_perf_map", ) - map_png_download( - m_nse, "perf", f"kaz_nse_map_lead{forecast_lead}d.png" - ) + map_png_download(m_nse, "perf", f"kaz_nse_map_lead{forecast_lead}d.png") _handle_map_click(perf_result, inv_filt, "_last_click_perf") st.caption( f"Selected basin: **{st.session_state.selected_gauge}** " diff --git a/scripts/replace_kaz_forcing.py b/scripts/replace_kaz_forcing.py index 205ed4b..d0d3e5a 100644 --- a/scripts/replace_kaz_forcing.py +++ b/scripts/replace_kaz_forcing.py @@ -119,7 +119,9 @@ def main() -> None: for p in args.new.glob("basin=*/scalar_dynamic.parquet") } logger.info( - "New static: %d basins | dynamic ERA5: %d basins", len(new_area), len(dyn_by_basin) + "New static: %d basins | dynamic ERA5: %d basins", + len(new_area), + len(dyn_by_basin), ) # Step 2: rewrite every timeseries partition @@ -131,7 +133,9 @@ def main() -> None: gid = d.name.replace("gauge_id=", "") path = d / "data.parquet" existing = pd.read_parquet(path) - kept = existing[[c for c in OBS_COLS if c in existing.columns] + ["date"]].copy() + kept = existing[ + [c for c in OBS_COLS if c in existing.columns] + ["date"] + ].copy() dyn_path = dyn_by_basin.get(gid) if dyn_path is not None: diff --git a/st_forecast/operational/sapphire_predictor.py b/st_forecast/operational/sapphire_predictor.py index e1aa1dd..4288fe0 100644 --- a/st_forecast/operational/sapphire_predictor.py +++ b/st_forecast/operational/sapphire_predictor.py @@ -38,9 +38,7 @@ def silence_pl_warnings() -> None: Importing :class:`SapphirePredictor` no longer mutates global logging state — that broke downstream apps that configured their own logging. """ - logging.getLogger("pytorch_lightning.utilities.rank_zero").setLevel( - logging.WARNING - ) + logging.getLogger("pytorch_lightning.utilities.rank_zero").setLevel(logging.WARNING) logging.getLogger("pytorch_lightning.accelerators.cuda").setLevel(logging.WARNING) @@ -76,7 +74,10 @@ def _validate_predict_inputs( target_col = config.target_col basin_id_col = config.basin_id_col - for name, df in (("past_measurements", past_measurements), ("covariates", covariates)): + for name, df in ( + ("past_measurements", past_measurements), + ("covariates", covariates), + ): if not isinstance(df, pd.DataFrame): raise TypeError( f"{name} must be a pandas DataFrame, got {type(df).__name__}." @@ -358,9 +359,7 @@ def predict( ) n = self.config.forecast_horizon code = _coerce_basin_code(identifier) - _validate_predict_inputs( - past_measurements, covariates, code, self.config - ) + _validate_predict_inputs(past_measurements, covariates, code, self.config) # Prepare DataFrames df_discharge, df_covariates, df_covariates_past = self._prepare_data( @@ -440,9 +439,7 @@ def hindcast( ) n = self.config.forecast_horizon code = _coerce_basin_code(identifier) - _validate_predict_inputs( - past_measurements, covariates, code, self.config - ) + _validate_predict_inputs(past_measurements, covariates, code, self.config) date_col = self.config.date_col # Prepare DataFrames (same as predict — no mode distinction) diff --git a/st_forecast/pipeline/artifacts.py b/st_forecast/pipeline/artifacts.py index a4b5e2a..e5004c3 100644 --- a/st_forecast/pipeline/artifacts.py +++ b/st_forecast/pipeline/artifacts.py @@ -109,9 +109,7 @@ class RunConfig: scheduler_eta_min: float = 1e-7 # For CosineAnnealingLR early_stopping_enabled: bool = True early_stopping_min_delta: float = 0.001 - likelihood_type: str = ( - "quantile" # "quantile" | "gaussian" | "gamma" | "gumbel" | "laplace" | "cauchy" | "mse" - ) + likelihood_type: str = "quantile" # "quantile" | "gaussian" | "gamma" | "gumbel" | "laplace" | "cauchy" | "mse" gradient_clip_val: float = 0.1 accelerator: str = "auto" checkpoint_strategy: str = "best" # "best" | "last" diff --git a/tests/test_public_api_smoke.py b/tests/test_public_api_smoke.py index 8b7b98d..144afa2 100644 --- a/tests/test_public_api_smoke.py +++ b/tests/test_public_api_smoke.py @@ -169,14 +169,18 @@ def test_rejects_missing_date_column(self): bad = self._good_df().rename(columns={"date": "timestamp"}) with pytest.raises(ValueError, match="missing required date column"): - _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + _validate_predict_inputs( + bad, self._good_df(), code=1, config=self._make_config() + ) def test_rejects_missing_target_column(self): from st_forecast.operational.sapphire_predictor import _validate_predict_inputs bad = self._good_df().drop(columns=["discharge"]) with pytest.raises(ValueError, match="missing target column"): - _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + _validate_predict_inputs( + bad, self._good_df(), code=1, config=self._make_config() + ) def test_warns_on_nan_in_target(self, caplog): """NaN in target is a warning, not an error: Darts skips hindcast @@ -186,8 +190,12 @@ def test_warns_on_nan_in_target(self, caplog): bad = self._good_df() bad.loc[2, "discharge"] = float("nan") - with caplog.at_level("WARNING", logger="st_forecast.operational.sapphire_predictor"): - _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + with caplog.at_level( + "WARNING", logger="st_forecast.operational.sapphire_predictor" + ): + _validate_predict_inputs( + bad, self._good_df(), code=1, config=self._make_config() + ) assert any( "contains 1 NaN value" in record.message for record in caplog.records ), f"expected NaN warning, got: {[r.message for r in caplog.records]}" @@ -198,7 +206,9 @@ def test_rejects_multi_basin_input(self): bad = self._good_df() bad.loc[0, "code"] = 999 with pytest.raises(ValueError, match="multiple basins"): - _validate_predict_inputs(bad, self._good_df(), code=1, config=self._make_config()) + _validate_predict_inputs( + bad, self._good_df(), code=1, config=self._make_config() + ) def _synthetic_basin_data( @@ -331,9 +341,9 @@ def test_train_save_reload_predict(self): # plus future covariates extending forecast_horizon days past it. target_code = 1002 # the larger-magnitude basin history_end = pd.Timestamp("2021-06-01") - past_window = ( - temporal_df["date"] <= history_end - ) & (temporal_df["code"] == target_code) + past_window = (temporal_df["date"] <= history_end) & ( + temporal_df["code"] == target_code + ) future_window = ( (temporal_df["date"] > history_end) & ( @@ -406,9 +416,9 @@ def test_predict_with_string_basin_id_still_unscales(self): predictor = SapphirePredictor(str(work_dir)) history_end = pd.Timestamp("2021-06-01") - past_window = ( - temporal_df["date"] <= history_end - ) & (temporal_df["code"] == 1002) + past_window = (temporal_df["date"] <= history_end) & ( + temporal_df["code"] == 1002 + ) future_window = ( (temporal_df["date"] > history_end) & ( @@ -512,8 +522,7 @@ def test_predictor_and_pipeline_hindcast_agree_on_q50(self): ["date", "code", "discharge"], ].copy() cv = temporal_df.loc[ - (temporal_df["code"] == target_code) - & (temporal_df["date"] <= cv_end), + (temporal_df["code"] == target_code) & (temporal_df["date"] <= cv_end), ["date", "code", "T", "P"], ].copy() @@ -532,9 +541,7 @@ def test_predictor_and_pipeline_hindcast_agree_on_q50(self): # Align on (date, forecast_step) — same model, same data, must match. join_keys = ["date", "forecast_step"] - merged = sp_hc.merge( - pl_hc, on=join_keys, suffixes=("_sp", "_pl") - ) + merged = sp_hc.merge(pl_hc, on=join_keys, suffixes=("_sp", "_pl")) assert len(merged) > 0, "no overlapping rows between the two hindcasts" # Q50 spread is dominated by sampling noise (num_samples=20 with @@ -582,8 +589,7 @@ def test_hindcast_schema_matches_pipeline_hindcast(self): ["date", "code", "discharge"], ].copy() cv = temporal_df.loc[ - (temporal_df["code"] == target_code) - & (temporal_df["date"] <= cv_end), + (temporal_df["code"] == target_code) & (temporal_df["date"] <= cv_end), ["date", "code", "T", "P"], ].copy() From 674b14ae1ac7b963e78b5f9fe9282246b83ea161 Mon Sep 17 00:00:00 2001 From: Sandro Hunziker <145295334+sandrohuni@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:50:58 +0200 Subject: [PATCH 20/20] fix: restrict predictor static covariates to configured features apply_static_scalers returns every static column, so SapphirePredictor attached all of them as static covariates while training filtered to config.static_features via filter_static_columns. Any static_df with extra columns (e.g. LAT/LON metadata) caused a component-count mismatch and failed every predict/hindcast call. Co-Authored-By: Claude Opus 4.8 (1M context) --- .bumpversion.toml | 2 +- pyproject.toml | 2 +- st_forecast/__init__.py | 2 +- st_forecast/operational/sapphire_predictor.py | 7 ++++++- uv.lock | 8 ++++++-- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index c7a23f9..4c7da55 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.1.10" +current_version = "0.1.11" commit = false tag = false parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" diff --git a/pyproject.toml b/pyproject.toml index 1bd3fdc..6453dd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "st-forecast" -version = "0.1.10" +version = "0.1.11" description = "Deep-learning short-term discharge forecasting for multi-basin hydrological systems; entry point to the SAPPHIRE forecast tools." readme = "README.md" requires-python = ">=3.11" diff --git a/st_forecast/__init__.py b/st_forecast/__init__.py index 80191c9..522c952 100644 --- a/st_forecast/__init__.py +++ b/st_forecast/__init__.py @@ -13,7 +13,7 @@ from st_forecast.pipeline.inference import DEFAULT_QUANTILES from st_forecast.pipeline.workflow import TrainModelResult, train_model -__version__ = "0.1.10" +__version__ = "0.1.11" # Package-level constants BASIN_ID_COL = "code" diff --git a/st_forecast/operational/sapphire_predictor.py b/st_forecast/operational/sapphire_predictor.py index 4288fe0..ab7ff6e 100644 --- a/st_forecast/operational/sapphire_predictor.py +++ b/st_forecast/operational/sapphire_predictor.py @@ -159,12 +159,17 @@ def __init__(self, model_path: str): self.config.static_features, self.config.static_id_col, ) - self.scaled_static_df = apply_static_scalers( + scaled_static_df = apply_static_scalers( self.static_df, self.scalers.static, static_features_list, self.config.static_scaler_type, ) + # Restrict to the configured static features so the attached + # covariates match the components the model was trained on. + # Training does this via filter_static_columns; apply_static_scalers + # returns every column, so filter here too. + self.scaled_static_df = scaled_static_df[static_features_list] else: self.scaled_static_df = None diff --git a/uv.lock b/uv.lock index 3efa44f..975a112 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.12'", @@ -781,6 +781,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, { url = "https://files.pythonhosted.org/packages/ae/8f/95d48d7e3d433e6dae5b1682e4292242a53f22df82e6d3dda81b1701a960/greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3", size = 644646, upload-time = "2025-08-07T13:45:26.523Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5e/405965351aef8c76b8ef7ad370e5da58d57ef6068df197548b015464001a/greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633", size = 640519, upload-time = "2025-08-07T13:53:13.928Z" }, { url = "https://files.pythonhosted.org/packages/25/5d/382753b52006ce0218297ec1b628e048c4e64b155379331f25a7316eb749/greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079", size = 639707, upload-time = "2025-08-07T13:18:27.146Z" }, { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, @@ -791,6 +792,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, @@ -801,6 +803,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, { url = "https://files.pythonhosted.org/packages/f7/0b/bc13f787394920b23073ca3b6c4a7a21396301ed75a655bcb47196b50e6e/greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc", size = 655191, upload-time = "2025-08-07T13:45:29.752Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d6/6adde57d1345a8d0f14d31e4ab9c23cfe8e2cd39c3baf7674b4b0338d266/greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a", size = 649516, upload-time = "2025-08-07T13:53:16.314Z" }, { url = "https://files.pythonhosted.org/packages/7f/3b/3a3328a788d4a473889a2d403199932be55b1b0060f4ddd96ee7cdfcad10/greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504", size = 652169, upload-time = "2025-08-07T13:18:32.861Z" }, { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, @@ -811,6 +814,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, { url = "https://files.pythonhosted.org/packages/c0/aa/687d6b12ffb505a4447567d1f3abea23bd20e73a5bed63871178e0831b7a/greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5", size = 699218, upload-time = "2025-08-07T13:45:30.969Z" }, + { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, { url = "https://files.pythonhosted.org/packages/23/6e/74407aed965a4ab6ddd93a7ded3180b730d281c77b765788419484cdfeef/greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269", size = 1612508, upload-time = "2025-11-04T12:42:23.427Z" }, @@ -3151,7 +3155,7 @@ wheels = [ [[package]] name = "st-forecast" -version = "0.1.10" +version = "0.1.11" source = { editable = "." } dependencies = [ { name = "click" },