diff --git a/docs/rekenmethodiek/geohydrologische_modellen.rst b/docs/rekenmethodiek/geohydrologische_modellen.rst index 4c3cb5c1..741fa121 100644 --- a/docs/rekenmethodiek/geohydrologische_modellen.rst +++ b/docs/rekenmethodiek/geohydrologische_modellen.rst @@ -34,7 +34,7 @@ verschilt. De grondwaterstroming vormt de drijvende kracht achter het proces van terugschrijdende erosie. Door het analytische grondwatermodel 4A van :cite:`trw_2004` toe te passen, kan de respons :math:`r_{exit}` in het uittredepunt worden beschreven als functie van de locatie in het -dwarspofiel :math:`x_{exit}` [m] en de geohydrologische parameters van model 4A. +dwarsprofiel :math:`x_{exit}` [m] en de geohydrologische parameters van model 4A. Een uitgebreide toelichting op de onderliggende theorie van dit model is te vinden in :ref:`stationair-model`. De respons in het uittredepunt wordt bepaald met: diff --git a/geoprob_pipe/app_object.py b/geoprob_pipe/app_object.py index ba827d89..574f0825 100644 --- a/geoprob_pipe/app_object.py +++ b/geoprob_pipe/app_object.py @@ -1,5 +1,6 @@ from __future__ import annotations import os +import sqlite3 from datetime import datetime from typing import TYPE_CHECKING, Optional, List import pandas as pd @@ -22,6 +23,8 @@ from geoprob_pipe.visualizations import Visualizations from geoprob_pipe.calculations.systems.build_and_run import build_and_run_system_calculations from geoprob_pipe.utils.update_metadata import update_metadata +from geoprob_pipe.utils.sql_contents import write_dfs_to_gpkg + import logging if TYPE_CHECKING: from geoprob_pipe.calculations.systems.build_and_run import CalcResult @@ -51,6 +54,14 @@ def __init__(self, app_settings: ApplicationSettings) -> None: self.time_start = datetime.now() self.input_data = InputData(app_settings=app_settings) # TODO: Alter with new option + + self.df_expanded = run_expand_input_tables(geopackage_filepath=self.input_data.app_settings.geopackage_filepath) + # TODO VJ: vind een betere plaats hiervoor + conn = sqlite3.connect(app_settings.geopackage_filepath) + df_add = self.df_expanded.copy() + df_add["parameter_input"] = df_add["parameter_input"].apply(str) + write_dfs_to_gpkg(conn, {"data__expanded_parameters": df_add}) + conn.close() # Read calculation settings # self._read_calculation_settings() # TODO: Not part of new version @@ -98,10 +109,10 @@ def export_archive(self): self.spatial.export_geopackage() # Parameter invoer als expanded - df_expanded = run_expand_input_tables(geopackage_filepath=self.input_data.app_settings.geopackage_filepath) + input_dir = os.path.join(export_dir, "input") os.makedirs(input_dir, exist_ok=True) - df_expanded.to_excel(excel_writer=os.path.join(input_dir, "df_parameter_invoer_expanded.xlsx"), index=False) + self.df_expanded.to_excel(excel_writer=os.path.join(input_dir, "df_parameter_invoer_expanded.xlsx"), index=False) # Input parameter figures obj = InputParameterFigures.populate( diff --git a/geoprob_pipe/calculations/systems/base_objects/base_system_build.py b/geoprob_pipe/calculations/systems/base_objects/base_system_build.py index 0a8c1f8e..ddc9ccb5 100644 --- a/geoprob_pipe/calculations/systems/base_objects/base_system_build.py +++ b/geoprob_pipe/calculations/systems/base_objects/base_system_build.py @@ -23,7 +23,7 @@ def _gather_variable_correlations(geopackage_filepath: str conn = sqlite3.connect(geopackage_filepath) cursor = conn.cursor() cursor.execute(""" -SELECT parameter_a, parameter_b, correlation FROM correlatie_invoer +SELECT parameter_a, parameter_b, correlation FROM data__correlatie_invoer WHERE correlation <> 0.0; """) rows = cursor.fetchall() # This will be a list of tuples diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index 112135d0..17e1d71a 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -148,7 +148,7 @@ def _validate_distributions_are_provided(): def _apply_distributions_on_variables(): for item in system_calculation.setup.distributions: name = item['name'] - + # TODO FIXME a_vak en delta_length geven een validation msg, deze worden niet in dit gebruikt. # Check if variable exists if system_calculation.results.reliability_project.variables[name] is None: system_calculation.validation_messages.add_warning( diff --git a/geoprob_pipe/calculations/systems/model4a/initial_input.py b/geoprob_pipe/calculations/systems/model4a/initial_input.py index a33a0940..44f74063 100644 --- a/geoprob_pipe/calculations/systems/model4a/initial_input.py +++ b/geoprob_pipe/calculations/systems/model4a/initial_input.py @@ -293,4 +293,22 @@ "mean": 0.3, "source": "excel", }, + { + "name": "a_vak", + "description": "De mechanisme gevoelige fractie van het element.", + "remark": "", + "unit": "-", + "distribution_type": DistributionType.deterministic, + "mean": 1.0, + "source": "excel" + }, + { + "name": "delta_length", + "description": "De equivalente onafhankelijke lengte voor STPH.", + "remark": "", + "unit": "m", + "distribution_type": DistributionType.deterministic, + "mean": 300.0, + "source": "excel" + }, ] diff --git a/geoprob_pipe/calculations/systems/moria/initial_input.py b/geoprob_pipe/calculations/systems/moria/initial_input.py index 877b9178..8adefc59 100644 --- a/geoprob_pipe/calculations/systems/moria/initial_input.py +++ b/geoprob_pipe/calculations/systems/moria/initial_input.py @@ -276,7 +276,7 @@ "remark": "", "unit": "kN/m³", "distribution_type": DistributionType.deterministic, - "mean": 16.5, + "mean": 26.0, "source": "excel", }, { @@ -288,4 +288,22 @@ "mean": 0.3, "source": "excel", }, + { + "name": "a_vak", + "description": "De mechanisme gevoelige fractie van het element.", + "remark": "", + "unit": "-", + "distribution_type": DistributionType.deterministic, + "mean": 1.0, + "source": "excel" + }, + { + "name": "delta_length", + "description": "De equivalente onafhankelijke lengte voor STPH.", + "remark": "", + "unit": "m", + "distribution_type": DistributionType.deterministic, + "mean": 300.0, + "source": "excel" + }, ] diff --git a/geoprob_pipe/calculations/systems/single_calc.py b/geoprob_pipe/calculations/systems/single_calc.py index 223bd83d..c33d4e74 100644 --- a/geoprob_pipe/calculations/systems/single_calc.py +++ b/geoprob_pipe/calculations/systems/single_calc.py @@ -42,7 +42,7 @@ def reproduce_single_calculation( conn = sqlite3.connect(geopackage_filepath) cursor = conn.cursor() cursor.execute( - 'SELECT "values" FROM geoprob_pipe_metadata WHERE metadata_type = ?', + 'SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?', ("geohydrologisch_model",)) geohydrologisch_model: str = cursor.fetchone()[0] diff --git a/geoprob_pipe/calculations/systems/validation.py b/geoprob_pipe/calculations/systems/validation.py index 23a13f78..c143590d 100644 --- a/geoprob_pipe/calculations/systems/validation.py +++ b/geoprob_pipe/calculations/systems/validation.py @@ -66,8 +66,8 @@ GAMMA_KORREL = ColumnValidation(column_name="mean", requirements=[ ValidationRequirement( requirement=requirements.is_in_range(left=23.0, right=29.0, inclusive="both"), - failure_msg=f"De parameter 'gamma_korrel' hoort 26.0 te zijn. " - f"De applicatie vereist dat deze tussen 23.0 en 29.0 is. ", + failure_msg="De parameter 'gamma_korrel' hoort 26.0 te zijn. " + "De applicatie vereist dat deze tussen 23.0 en 29.0 is. ", filters=filters.combine(filters.is_in(column="parameter", values=["gamma_korrel"]), FILTER_IS_NOT_CDF_CURVE))]) diff --git a/geoprob_pipe/calculations/systems/wbi/initial_input.py b/geoprob_pipe/calculations/systems/wbi/initial_input.py index d3285ac1..5381d75d 100644 --- a/geoprob_pipe/calculations/systems/wbi/initial_input.py +++ b/geoprob_pipe/calculations/systems/wbi/initial_input.py @@ -251,4 +251,22 @@ "mean": 0.3, "source": "excel", }, + { + "name": "a_vak", + "description": "De mechanisme gevoelige fractie van het element.", + "remark": "", + "unit": "-", + "distribution_type": DistributionType.deterministic, + "mean": 1.0, + "source": "excel" + }, + { + "name": "delta_length", + "description": "De equivalente onafhankelijke lengte voor STPH.", + "remark": "", + "unit": "m", + "distribution_type": DistributionType.deterministic, + "mean": 300.0, + "source": "excel" + }, ] diff --git a/geoprob_pipe/cmd_app/cmd.py b/geoprob_pipe/cmd_app/cmd.py index 8fd5d9e8..ec8c7e8b 100644 --- a/geoprob_pipe/cmd_app/cmd.py +++ b/geoprob_pipe/cmd_app/cmd.py @@ -1,33 +1,49 @@ -import typer -from rich.console import Console import os import sqlite3 -from geoprob_pipe.utils import clear_terminal +from pathlib import Path +from configparser import ConfigParser from datetime import datetime +from importlib.metadata import distributions +from typing import Dict, List, Optional + +import typer +from rich.console import Console from rich.panel import Panel + from geoprob_pipe.cmd_app.questionnaire import start_questionnaire -from typing import Optional, List, Dict from geoprob_pipe.cmd_app.utils.misc import get_geoprob_pipe_version_number +from geoprob_pipe.utils import clear_terminal from geoprob_pipe.utils.loggers import setup_base_logging -from importlib.metadata import distributions app = typer.Typer(help="GeoProb-Pipe - CLI applicatie voor probabilistische piping berekeningen.", add_completion=False) class ApplicationSettings: - + def __init__(self): - self.workspace_dir: Optional[str] = None - self.geopackage_filename: Optional[str] = None + self.workspace_dir: Path = Path() + self.geopackage_filename: Path = Path() self.datetime_stamp: str = datetime.now().strftime("%Y-%m-%d_%H%M%S") self.to_run = "all" # -> or vakken:1,2,3,4,5 self.debug: bool = os.getenv("GEOPROB_DEBUG") == "1" + # Flags + self.batch_input: bool = False # Flag for batch input @property def geopackage_filepath(self) -> str: return os.path.join(self.workspace_dir, self.geopackage_filename) + + @property + def batch_input_filepath(self) -> str: + return os.path.join(self.workspace_dir, "batch_input.ini") + @property + def input_config(self): + config = ConfigParser() + config.read(self.batch_input_filepath) + return config + @property def hrd_dir(self): path_to_hrd_dir = os.path.join(self.workspace_dir, "hrd_files") @@ -61,7 +77,7 @@ def geohydrologisch_model(self) -> str: conn = sqlite3.connect(self.geopackage_filepath) cursor = conn.cursor() cursor.execute(""" - SELECT geoprob_pipe_metadata."values" + SELECT geoprob_pipe_metadata.metadata_value FROM geoprob_pipe_metadata WHERE metadata_type='geohydrologisch_model'; """) @@ -103,7 +119,7 @@ def startup_geoprob_pipe(): debug_label: str = "" if app_settings.debug: - debug_label = f", DEBUG=TRUE" + debug_label = ", DEBUG=TRUE" clear_terminal() console = Console() diff --git a/geoprob_pipe/cmd_app/comparisons/__init__.py b/geoprob_pipe/cmd_app/comparisons/__init__.py index 953904e7..c9485fb1 100644 --- a/geoprob_pipe/cmd_app/comparisons/__init__.py +++ b/geoprob_pipe/cmd_app/comparisons/__init__.py @@ -54,41 +54,41 @@ def _load_result_data_from_geopackage(self): """Method om de data te verzamelen uit de opgegeven pakketten. Raises: - ValueError: Als de `beta_limit_states` tabellen niet hetzelfde + ValueError: Als de `result__limit_states` tabellen niet hetzelfde formaat hebben. - ValueError: Als de `beta_scenario` tabellen niet hetzelfde + ValueError: Als de `result__scenario` tabellen niet hetzelfde formaat hebben. - ValueError: Als de `beta_uitredepunten` tabellen niet hetzelfde + ValueError: Als de `result__uitredepunten` tabellen niet hetzelfde formaat hebben. """ conn_1 = sqlite3.connect(self.geopackage_filepath_1) conn_2 = sqlite3.connect(self.geopackage_filepath_2) self.df1_beta_limit_states = pd.read_sql( - "SELECT * FROM beta_limit_states;", conn_1 + "SELECT * FROM result__limit_states;", conn_1 ) self.df2_beta_limit_states = pd.read_sql( - "SELECT * FROM beta_limit_states;", conn_2 + "SELECT * FROM result__limit_states;", conn_2 ) if len(self.df1_beta_limit_states) != len(self.df2_beta_limit_states): raise ValueError("De beta_limit_states tables hebben niet " "hetzelfde formaat") self.df1_beta_scenarios = pd.read_sql( - "SELECT * FROM beta_scenarios_final;", conn_1 + "SELECT * FROM result__scenarios_final;", conn_1 ) self.df2_beta_scenarios = pd.read_sql( - "SELECT * FROM beta_scenarios_final;", conn_2 + "SELECT * FROM result__scenarios_final;", conn_2 ) if len(self.df1_beta_scenarios) != len(self.df2_beta_scenarios): raise ValueError("De beta_scenario tables hebben niet hetzelfde " "formaat") self.df1_beta_uittredepunten = pd.read_sql( - "SELECT * FROM beta_uittredepunten;", conn_1 + "SELECT * FROM result__uittredepunten;", conn_1 ) self.df2_beta_uittredepunten = pd.read_sql( - "SELECT * FROM beta_uittredepunten;", conn_2 + "SELECT * FROM result__uittredepunten;", conn_2 ) if len(self.df1_beta_uittredepunten) != len(self.df2_beta_uittredepunten): raise ValueError("De beta_uittredepunten tables hebben niet " @@ -108,11 +108,11 @@ def _load_uittredepunten_gdf(self): """ self.gdf1_uittredepunten = gpd.read_file( self.geopackage_filepath_1, - layer="beta_uittredepunten" + layer="result__uittredepunten" ) self.gdf2_uittredepunten = gpd.read_file( self.geopackage_filepath_2, - layer="beta_uittredepunten" + layer="result__uittredepunten" ) if len(self.gdf1_uittredepunten) != len(self.gdf2_uittredepunten): raise ValueError("De beta_uittredepunten tables hebben niet " diff --git a/geoprob_pipe/cmd_app/comparisons/beta_map.py b/geoprob_pipe/cmd_app/comparisons/beta_map.py index a153b2b8..d3c6b9db 100644 --- a/geoprob_pipe/cmd_app/comparisons/beta_map.py +++ b/geoprob_pipe/cmd_app/comparisons/beta_map.py @@ -1,9 +1,13 @@ from __future__ import annotations + import os -import plotly.graph_objects as go -import geopandas as gpd -from shapely.geometry import LineString, MultiLineString, GeometryCollection from typing import TYPE_CHECKING + +import fiona +import geopandas as gpd +import plotly.graph_objects as go +from shapely.geometry import GeometryCollection, LineString, MultiLineString + if TYPE_CHECKING: from geoprob_pipe.cmd_app.comparisons import ComparisonCollector @@ -35,59 +39,72 @@ def _calculate_zoom(lat_range_val, lon_range_val): return zoom -def _add_line(comparison: ComparisonCollector, fig: go.Figure, - layer: str, color: str): - gdf_traject = gpd.read_file(comparison.geopackage_filepath_1, - layer=layer) - gdf_traject = gdf_traject.to_crs("EPSG:4326") - - def plot_linestring(ls, display): - xs, ys = ls.xy - xs = list(xs) - ys = list(ys) - fig.add_trace(go.Scattermap( - lon=xs, - lat=ys, - mode="lines", - line=dict(color=color, width=1.5), - hoverinfo="none", - name=layer, - legendgroup=layer, - showlegend=display - )) - show = True - for geom in gdf_traject.geometry: - if isinstance(geom, LineString): - plot_linestring(geom, show) - show = False - elif isinstance(geom, MultiLineString): - for line in geom.geoms: - plot_linestring(line, show) +def _add_line( + comparison: ComparisonCollector, fig: go.Figure, layer: str, color: str +): + gpkg_layers: list[str] = fiona.listlayers(comparison.geopackage_filepath_1) + list_layers: list[str] = [ + lay for lay in gpkg_layers if lay.split("__")[0] == layer + ] + + for listed_layer in list_layers: + gdf_traject = gpd.read_file( + comparison.geopackage_filepath_1, layer=listed_layer + ) + gdf_traject = gdf_traject.to_crs("EPSG:4326") + + def plot_linestring(ls, display): + xs, ys = ls.xy + xs = list(xs) + ys = list(ys) + fig.add_trace( + go.Scattermap( + lon=xs, + lat=ys, + mode="lines", + line=dict(color=color, width=1.5), + hoverinfo="none", + name=listed_layer, + legendgroup=layer, + showlegend=display, + ) + ) + + show = True + for geom in gdf_traject.geometry: + if isinstance(geom, LineString): + plot_linestring(geom, show) show = False - elif isinstance(geom, GeometryCollection): - for g in geom.geoms: - if isinstance(g, LineString): - plot_linestring(g, show) + elif isinstance(geom, MultiLineString): + for line in geom.geoms: + plot_linestring(line, show) show = False - elif isinstance(g, MultiLineString): - for line in g.geoms: - plot_linestring(line, show) + elif isinstance(geom, GeometryCollection): + for g in geom.geoms: + if isinstance(g, LineString): + plot_linestring(g, show) show = False + elif isinstance(g, MultiLineString): + for line in g.geoms: + plot_linestring(line, show) + show = False - else: - print("Skipping unsupported geometry:", geom.geom_type) + else: + print("Skipping unsupported geometry:", geom.geom_type) return fig -def map_delta_beta_comparison(comparison: ComparisonCollector, - export: bool = False) -> go.Figure: +def map_delta_beta_comparison( + comparison: ComparisonCollector, export: bool = False +) -> go.Figure: # load data from class - gdf_result1 = (comparison.gdf1_uittredepunten[ + gdf_result1 = comparison.gdf1_uittredepunten[ ["uittredepunt_id", "beta", "geometry"] - ].rename(columns={"beta": "beta1"})) - gdf_result2 = (comparison.gdf2_uittredepunten[ - ["uittredepunt_id", "beta"]].rename(columns={"beta": "beta2"})) + ].rename(columns={"beta": "beta1"}) + gdf_result2 = comparison.gdf2_uittredepunten[ + ["uittredepunt_id", "beta"] + ].rename(columns={"beta": "beta2"}) gdf = gdf_result1.merge(gdf_result2, on="uittredepunt_id") gdf["beta_delta"] = gdf["beta2"] - gdf["beta1"] @@ -95,37 +112,39 @@ def map_delta_beta_comparison(comparison: ComparisonCollector, gdf_latlon = gdf.to_crs("EPSG:4326") fig = go.Figure() - fig.add_trace(go.Scattermap( - mode="markers", - lat=gdf_latlon.geometry.y, - lon=gdf_latlon.geometry.x, - marker=dict( - size=9, - color="black" - ), - showlegend=False - )) + fig.add_trace( + go.Scattermap( + mode="markers", + lat=gdf_latlon.geometry.y, + lon=gdf_latlon.geometry.x, + marker=dict(size=9, color="black"), + showlegend=False, + ) + ) hoverdata = ["uittredepunt_id", "beta_delta", "beta1", "beta2"] - fig.add_trace(go.Scattermap( - mode="markers", - lat=gdf_latlon.geometry.y, - lon=gdf_latlon.geometry.x, - marker=dict( - size=8, - color=gdf_latlon["beta_delta"], - cmax=3, - cmin=-3, - colorscale="RdYlGn", - colorbar=dict(title="Delta Beta") - ), - hoverinfo='text', - text=gdf_latlon[hoverdata].apply( - lambda row: '
'.join( - [f"{col}: {round(row[col], 3)}" for col in hoverdata] + fig.add_trace( + go.Scattermap( + mode="markers", + lat=gdf_latlon.geometry.y, + lon=gdf_latlon.geometry.x, + marker=dict( + size=8, + color=gdf_latlon["beta_delta"], + cmax=3, + cmin=-3, + colorscale="RdYlGn", + colorbar=dict(title="Delta Beta"), + ), + hoverinfo="text", + text=gdf_latlon[hoverdata].apply( + lambda row: "
".join( + [f"{col}: {round(row[col], 3)}" for col in hoverdata] ), - axis=1), - showlegend=False - )) + axis=1, + ), + showlegend=False, + ) + ) fig = _add_line(comparison, fig, "dijktraject", "black") fig = _add_line(comparison, fig, "intredelijn", "blue") @@ -138,24 +157,23 @@ def map_delta_beta_comparison(comparison: ComparisonCollector, # carto-positron, open-street-map, satellite-streets map_zoom=zoom, map_center=dict( - lat=gdf_latlon.geometry.y.mean(), - lon=gdf_latlon.geometry.x.mean() + lat=gdf_latlon.geometry.y.mean(), lon=gdf_latlon.geometry.x.mean() ), dragmode="zoom", - title=f"Delta beta van uittredepunten tussen
" + - f"Beta1: {comparison.name_1} en Beta2: {comparison.name_2}", + title="Delta beta van uittredepunten tussen
" + + f"Beta1: {comparison.name_1} en Beta2: {comparison.name_2}", legend=dict( - orientation="h", - yanchor="bottom", - y=1.02, - xanchor="right", - x=1)) + orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1 + ), + ) if export: os.makedirs(comparison.export_dir, exist_ok=True) - fig.write_html(os.path.join( - comparison.export_dir, "delta_beta_map.html" - ), include_plotlyjs='cdn', include_mathjax='cdn') + fig.write_html( + os.path.join(comparison.export_dir, "delta_beta_map.html"), + include_plotlyjs="cdn", + include_mathjax="cdn", + ) # fig.write_image(os.path.join( # comparison.export_dir, "delta_beta_map.png" # ), format="png", scale=5, width=1400) @@ -163,14 +181,16 @@ def map_delta_beta_comparison(comparison: ComparisonCollector, return fig -def map_ratio_beta_comparison(comparison: ComparisonCollector, - export: bool = False) -> go.Figure: +def map_ratio_beta_comparison( + comparison: ComparisonCollector, export: bool = False +) -> go.Figure: # load data from class - gdf_result1 = (comparison.gdf1_uittredepunten[ + gdf_result1 = comparison.gdf1_uittredepunten[ ["uittredepunt_id", "beta", "geometry"] - ].rename(columns={"beta": "beta1"})) - gdf_result2 = (comparison.gdf2_uittredepunten[ - ["uittredepunt_id", "beta"]].rename(columns={"beta": "beta2"})) + ].rename(columns={"beta": "beta1"}) + gdf_result2 = comparison.gdf2_uittredepunten[ + ["uittredepunt_id", "beta"] + ].rename(columns={"beta": "beta2"}) gdf = gdf_result1.merge(gdf_result2, on="uittredepunt_id") gdf["beta_ratio"] = round((gdf["beta1"] / gdf["beta2"]) * 100, 2) @@ -179,37 +199,39 @@ def map_ratio_beta_comparison(comparison: ComparisonCollector, gdf_latlon = gdf.to_crs("EPSG:4326") fig = go.Figure() - fig.add_trace(go.Scattermap( - mode="markers", - lat=gdf_latlon.geometry.y, - lon=gdf_latlon.geometry.x, - marker=dict( - size=9, - color="black" - ), - showlegend=False - )) + fig.add_trace( + go.Scattermap( + mode="markers", + lat=gdf_latlon.geometry.y, + lon=gdf_latlon.geometry.x, + marker=dict(size=9, color="black"), + showlegend=False, + ) + ) hoverdata = ["uittredepunt_id", "beta_ratio", "beta1", "beta2"] - fig.add_trace(go.Scattermap( - mode="markers", - lat=gdf_latlon.geometry.y, - lon=gdf_latlon.geometry.x, - marker=dict( - size=8, - color=gdf_latlon["beta_ratio"], - cmax=200, - cmin=0, - colorscale="RdYlGn", - colorbar=dict(title="Beta Ratio - Beta1/Beta2 [%]"), - ), - hoverinfo='text', - text=gdf_latlon[hoverdata].apply( - lambda row: '
'.join( - [f"{col}: {round(row[col], 3)}" for col in hoverdata] + fig.add_trace( + go.Scattermap( + mode="markers", + lat=gdf_latlon.geometry.y, + lon=gdf_latlon.geometry.x, + marker=dict( + size=8, + color=gdf_latlon["beta_ratio"], + cmax=200, + cmin=0, + colorscale="RdYlGn", + colorbar=dict(title="Beta Ratio - Beta1/Beta2 [%]"), + ), + hoverinfo="text", + text=gdf_latlon[hoverdata].apply( + lambda row: "
".join( + [f"{col}: {round(row[col], 3)}" for col in hoverdata] ), - axis=1), - showlegend=False - )) + axis=1, + ), + showlegend=False, + ) + ) fig = _add_line(comparison, fig, "dijktraject", "black") fig = _add_line(comparison, fig, "intredelijn", "blue") @@ -222,25 +244,22 @@ def map_ratio_beta_comparison(comparison: ComparisonCollector, # carto-positron, open-street-map, satellite-streets map_zoom=zoom, map_center=dict( - lat=gdf_latlon.geometry.y.mean(), - lon=gdf_latlon.geometry.x.mean() + lat=gdf_latlon.geometry.y.mean(), lon=gdf_latlon.geometry.x.mean() ), dragmode="zoom", - title=f"Beta ratio van uittredepunten tussen
" + - f"Beta1: {comparison.name_1} en Beta2: {comparison.name_2}", + title="Beta ratio van uittredepunten tussen
" + + f"Beta1: {comparison.name_1} en Beta2: {comparison.name_2}", legend=dict( - orientation="h", - yanchor="bottom", - y=1.02, - xanchor="right", - x=1 - ) - ) + orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1 + ), + ) if export: os.makedirs(comparison.export_dir, exist_ok=True) - fig.write_html(os.path.join( - comparison.export_dir, "ratio_beta_map.html" - ), include_plotlyjs='cdn', include_mathjax='cdn') + fig.write_html( + os.path.join(comparison.export_dir, "ratio_beta_map.html"), + include_plotlyjs="cdn", + include_mathjax="cdn", + ) # fig.write_image(os.path.join( # comparison.export_dir, "ratio_beta_map.png" # ), format="png", scale=5, width=1400) diff --git a/geoprob_pipe/cmd_app/comparisons/start_comparison.py b/geoprob_pipe/cmd_app/comparisons/start_comparison.py index bea5b14e..39d4c01b 100644 --- a/geoprob_pipe/cmd_app/comparisons/start_comparison.py +++ b/geoprob_pipe/cmd_app/comparisons/start_comparison.py @@ -13,9 +13,7 @@ def specify_dir_for_first_file(): Returns: filepath: """ - filepath: str = "" - filepath_is_valid = False - while filepath_is_valid is False: + while True: filepath: str = inq_text( message=("Specificeer het volledige bestandspad naar het eerste " ".geoprob_pipe.gpkg-bestand."), @@ -35,7 +33,7 @@ def specify_dir_for_first_file(): BColors.ENDC) continue - filepath_is_valid = True + break return filepath @@ -47,9 +45,7 @@ def specify_dir_for_second_file(): Returns: filepath: """ - filepath: str = "" - filepath_is_valid = False - while filepath_is_valid is False: + while True: filepath: str = inq_text( message=("Specificeer het volledige bestandspad naar het tweede " ".geoprob_pipe.gpkg-bestand."), @@ -64,12 +60,13 @@ def specify_dir_for_second_file(): "deze extensie.", BColors.ENDC) continue + if not os.path.exists(filepath): print(BColors.WARNING, "Het opgegeven bestandspad bestaat niet.", BColors.ENDC) continue - filepath_is_valid = True + break return filepath @@ -81,9 +78,7 @@ def specify_dir_for_comparison(): Returns: workspace_dir: """ - workspace_dir: str = "" - workspace_dir_is_valid = False - while workspace_dir_is_valid is False: + while True: workspace_dir: str = inq_text( message="Specificeer het volledige pad naar de map waar je" + " de export van de vergelijking wilt opslaan.", @@ -94,12 +89,13 @@ def specify_dir_for_comparison(): print(BColors.WARNING, "De opgegeven map bestaat niet.", BColors.ENDC) continue + if not os.path.isdir(workspace_dir): print(BColors.WARNING, "De opgegeven locatie is geen map.", BColors.ENDC) continue - workspace_dir_is_valid = True + break return workspace_dir diff --git a/geoprob_pipe/cmd_app/general/geohydrologisch_model.py b/geoprob_pipe/cmd_app/general/geohydrologisch_model.py index 1bcb81e0..6e983a3f 100644 --- a/geoprob_pipe/cmd_app/general/geohydrologisch_model.py +++ b/geoprob_pipe/cmd_app/general/geohydrologisch_model.py @@ -20,7 +20,7 @@ def created_model(app_settings: ApplicationSettings) -> bool: return True # Check if specified model is legal - current_specified_model = df[df['metadata_type'] == "geohydrologisch_model"]["values"].iloc[0] + current_specified_model = df[df['metadata_type'] == "geohydrologisch_model"]["metadata_value"].iloc[0] if current_specified_model not in CALCULATION_MAPPER.keys(): specify_model_to_use(app_settings, update_record=True) return True @@ -46,11 +46,11 @@ def specify_model_to_use(app_settings: ApplicationSettings, update_record: bool choice_formatted = choice_formatted.replace(" ", "").lower() sql_statement = f""" INSERT INTO geoprob_pipe_metadata - ('metadata_type', 'values') VALUES ('geohydrologisch_model', '{choice_formatted}')""" + (metadata_type, metadata_value) VALUES ('geohydrologisch_model', '{choice_formatted}')""" if update_record: sql_statement = f""" UPDATE geoprob_pipe_metadata - SET 'values' = '{choice_formatted}' + SET metadata_value = '{choice_formatted}' WHERE metadata_type = 'geohydrologisch_model'""" cursor.execute(sql_statement) conn.commit() diff --git a/geoprob_pipe/cmd_app/general/project.py b/geoprob_pipe/cmd_app/general/project.py index 68fbe4d0..64b2f4e9 100644 --- a/geoprob_pipe/cmd_app/general/project.py +++ b/geoprob_pipe/cmd_app/general/project.py @@ -65,7 +65,7 @@ def created_project(app_settings: ApplicationSettings) -> bool: console = Console() console.print(Panel( EXPLANATION_REPRODUCING_SINGLE_CALCULATION, - title=f"Inspecteer een enkele berekening".upper(), + title="Inspecteer een enkele berekening".upper(), title_align="left", border_style="bright_blue", padding=(0, 2))) @@ -78,7 +78,7 @@ def created_project(app_settings: ApplicationSettings) -> bool: def _get_file_geoprob_pipe_version(file_path: str) -> Optional[str]: conn = sqlite3.connect(file_path) cursor = conn.cursor() - cursor.execute(f"SELECT * FROM geoprob_pipe_metadata WHERE metadata_type = 'pip_freeze' LIMIT 1;") + cursor.execute("SELECT * FROM geoprob_pipe_metadata WHERE metadata_type = 'pip_freeze' LIMIT 1;") row = cursor.fetchone() dict_item = ast.literal_eval(row[2]) @@ -97,10 +97,10 @@ def _compare_versions_and_possibly_warn(installed_version: str, file_version: st version_diff_msg = ( f"Er is een GeoProb-Pipe versie verschil tussen de installatie ({v_installed}) en het bestand ({v_file}).") reinstall_msg = ( - f"\n" - f"Wil je de versies liever gelijk trekken? Dit doe je door eerst de applicatie af te sluiten en vervolgens het \n" - f"commando `pip install geoprob-pipe==versie_nummer` uit te voeren. Waarbij je `versie_nummer` vervangt met het \n" - f"nummer van de gewenste versie") + "\n" + "Wil je de versies liever gelijk trekken? Dit doe je door eerst de applicatie af te sluiten en vervolgens het \n" + "commando `pip install geoprob-pipe==versie_nummer` uit te voeren. Waarbij je `versie_nummer` vervangt met het \n" + "nummer van de gewenste versie") if v_installed.major != v_file.major: print(f"\n{BColors.FAIL}SEVERE WARNING:\n" @@ -109,6 +109,7 @@ def _compare_versions_and_possibly_warn(installed_version: str, file_version: st f"compatibel met elkaar zijn. \n" f"{reinstall_msg}" f"{BColors.ENDC}\n") + elif v_installed.minor != v_file.minor: print(f"\n{BColors.WARNING}WARNING:\n" f"{version_diff_msg}\n" @@ -116,6 +117,7 @@ def _compare_versions_and_possibly_warn(installed_version: str, file_version: st f"de applicatie. \n" f"{reinstall_msg}" f"{BColors.ENDC}\n") + elif v_installed.micro != v_file.micro: print(f"\n{BColors.OKBLUE}" f"{version_diff_msg}\n" @@ -139,9 +141,7 @@ def _possibly_warn_if_version_difference(file_path: str): def specify_path_to_existing_project(app_settings: ApplicationSettings): - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: + while True: filepath: str = inquirer.text( message="Specificeer het volledige bestandspad naar het .geoprob_pipe.gpkg-bestand.", ).execute() @@ -153,45 +153,42 @@ def specify_path_to_existing_project(app_settings: ApplicationSettings): f"{os.path.basename(filepath)} eindigt niet op deze extensie.", BColors.ENDC) continue if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) + print(BColors.WARNING, "Het opgegeven bestandspad bestaat niet.", BColors.ENDC) continue - filepath_is_valid = True + break - app_settings.workspace_dir = os.path.dirname(filepath) - app_settings.geopackage_filename = os.path.basename(filepath) + app_settings.workspace_dir = Path(os.path.dirname(filepath)) + app_settings.geopackage_filename = Path(os.path.basename(filepath)) _possibly_warn_if_version_difference(file_path=app_settings.geopackage_filepath) def specify_dir_for_new_project(app_settings: ApplicationSettings): - workspace_dir: Optional[str] = None - workspace_dir_is_valid = False - while workspace_dir_is_valid is False: + while True: workspace_dir: str = inquirer.text( message="Specificeer het volledige pad naar de map waar je het GeoProb-Pipe-bestand wilt opslaan.", ).execute() workspace_dir = workspace_dir.replace('"', '') if not os.path.exists(workspace_dir): - print(BColors.WARNING, f"De opgegeven map bestaat niet.", BColors.ENDC) - continue + print(BColors.WARNING, "De opgegeven map bestaat niet.", BColors.ENDC) + + if not os.path.isdir(workspace_dir): - print(BColors.WARNING, f"De opgegeven locatie is geen map.", BColors.ENDC) + print(BColors.WARNING, "De opgegeven locatie is geen map.", BColors.ENDC) continue - workspace_dir_is_valid = True + break - app_settings.workspace_dir = workspace_dir + app_settings.workspace_dir = Path(workspace_dir) # Continue questionnaire specify_project_filename(app_settings) def specify_project_filename(app_settings: ApplicationSettings): - filename: Optional[str] = None - filename_is_valid = False - while filename_is_valid is False: + while True: project_name: str = inquirer.text( message="Specificeer een bestandsnaam voor het project. " "Het bestand wordt opgeslagen met een .geoprob_pipe.gpkg-extensie.", @@ -207,9 +204,9 @@ def specify_project_filename(app_settings: ApplicationSettings): f" {filepath}", BColors.ENDC) continue - filename_is_valid = True + break - app_settings.geopackage_filename = filename + app_settings.geopackage_filename = Path(filename) create_geopackage_file(app_settings) @@ -218,7 +215,7 @@ def create_geopackage_file(app_settings: ApplicationSettings): df = DataFrame({ "metadata_type": ["created_by", "created_datetime", "application_version", "python_version", "pip_freeze"], - "values": [ + "metadata_value": [ os.getenv("USERNAME"), datetime.now(), get_geoprob_pipe_version_number(), diff --git a/geoprob_pipe/cmd_app/general/traject_parameters.py b/geoprob_pipe/cmd_app/general/traject_parameters.py index c51ee9e0..aabafa34 100644 --- a/geoprob_pipe/cmd_app/general/traject_parameters.py +++ b/geoprob_pipe/cmd_app/general/traject_parameters.py @@ -1,10 +1,14 @@ from __future__ import annotations -from geoprob_pipe.utils.validation_messages import BColors -from InquirerPy import inquirer -from typing import TYPE_CHECKING, Optional -from geopandas import read_file + import sqlite3 +from typing import TYPE_CHECKING + +from geopandas import read_file +from InquirerPy import inquirer from pandas import DataFrame + +from geoprob_pipe.utils.validation_messages import BColors + if TYPE_CHECKING: from geoprob_pipe.cmd_app.cmd import ApplicationSettings @@ -13,29 +17,24 @@ def _append_to_db(app_settings: ApplicationSettings, key: str, value): file_path = app_settings.geopackage_filepath conn = sqlite3.connect(file_path) cursor = conn.cursor() - cursor.execute( - """ - INSERT INTO geoprob_pipe_metadata (metadata_type, "values") VALUES (?, ?); - """, - (key, value)) + cursor.execute("INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES (?, ?);", + (key, value)) conn.commit() cursor.close() def _specify_traject_id(app_settings: ApplicationSettings): - traject_id: Optional[str] = None - traject_id_is_valid = False - while traject_id_is_valid is False: + while True: traject_id: str = inquirer.text( message="Specificeer een tekstuele identificatie van het dijktraject die je graag gebruikt. ", ).execute() traject_id = traject_id.strip() - if traject_id is "": - print(BColors.OKBLUE, f"Je hebt geen traject_id gespecificeerd. Je invoer is leeg. ", BColors.ENDC) + if traject_id == "": + print(BColors.OKBLUE, "Je hebt geen traject_id gespecificeerd. Je invoer is leeg. ", BColors.ENDC) continue - traject_id_is_valid = True + break _append_to_db(app_settings=app_settings, key='traject_id', value=traject_id) @@ -48,52 +47,49 @@ def is_integer(s: str) -> bool: def _specify_signaleringswaarde(app_settings: ApplicationSettings): - signaleringswaarde_int: Optional[int] = None - signaleringswaarde_is_valid = False - while signaleringswaarde_is_valid is False: + while True: signaleringswaarde: str = inquirer.text( message="Specificeer de signaleringswaarde (geheel getal boven 10). ", ).execute() signaleringswaarde = signaleringswaarde.strip() - if signaleringswaarde is "": - print(BColors.OKBLUE, f"Je hebt geen signaleringswaarde gespecificeerd. Je invoer is leeg. ", BColors.ENDC) + if signaleringswaarde == "": + print(BColors.OKBLUE, "Je hebt geen signaleringswaarde gespecificeerd. Je invoer is leeg. ", BColors.ENDC) continue if not is_integer(s=signaleringswaarde): - print(BColors.OKBLUE, f"Je ingevoerde signaleringswaarde is geen geheel getal.", BColors.ENDC) + print(BColors.OKBLUE, "Je ingevoerde signaleringswaarde is geen geheel getal.", BColors.ENDC) continue signaleringswaarde_int = int(signaleringswaarde) if signaleringswaarde_int < 10: - print(BColors.OKBLUE, f"Je ingevoerde signaleringswaarde is kleiner dan 10. ", BColors.ENDC) + print(BColors.OKBLUE, "Je ingevoerde signaleringswaarde is kleiner dan 10. ", BColors.ENDC) continue - signaleringswaarde_is_valid = True + break _append_to_db(app_settings=app_settings, key='signaleringswaarde', value=signaleringswaarde_int) def _specify_ondergrens(app_settings: ApplicationSettings, signaleringswaarde: int): - ondergrens_int: Optional[int] = None - ondergrens_is_valid = False - while ondergrens_is_valid is False: + while True: ondergrens: str = inquirer.text( message="Specificeer de ondergrens (geheel getal boven 10). ", ).execute() ondergrens = ondergrens.strip() - if ondergrens is "": - print(BColors.OKBLUE, f"Je hebt geen ondergrens gespecificeerd. Je invoer is leeg. ", BColors.ENDC) + if ondergrens == "": + print(BColors.OKBLUE, "Je hebt geen ondergrens gespecificeerd. Je invoer is leeg. ", BColors.ENDC) continue if not is_integer(s=ondergrens): - print(BColors.OKBLUE, f"Je ingevoerde ondergrens is geen geheel getal.", BColors.ENDC) + print(BColors.OKBLUE, "Je ingevoerde ondergrens is geen geheel getal.", BColors.ENDC) continue + ondergrens_int = int(ondergrens) if ondergrens_int < 10: - print(BColors.OKBLUE, f"Je ingevoerde ondergrens is kleiner dan 10. ", BColors.ENDC) + print(BColors.OKBLUE, "Je ingevoerde ondergrens is kleiner dan 10. ", BColors.ENDC) continue if ondergrens_int > signaleringswaarde: @@ -102,7 +98,7 @@ def _specify_ondergrens(app_settings: ApplicationSettings, signaleringswaarde: i BColors.ENDC) continue - ondergrens_is_valid = True + break _append_to_db(app_settings=app_settings, key='ondergrens', value=ondergrens_int) @@ -111,7 +107,7 @@ def _get_signaleringswaarde(app_settings: ApplicationSettings) -> int: conn = sqlite3.connect(app_settings.geopackage_filepath) cursor = conn.cursor() cursor.execute(""" - SELECT geoprob_pipe_metadata."values" + SELECT geoprob_pipe_metadata.metadata_value FROM geoprob_pipe_metadata WHERE metadata_type='signaleringswaarde'; """) @@ -133,34 +129,33 @@ def is_float(s: str) -> bool: def _specify_w(app_settings: ApplicationSettings): - w_float: Optional[float] = None - w_is_valid = False - while w_is_valid is False: + + while True: w: str = inquirer.text( message="Specificeer de w (decimaal getal tussen 0.0 en 1.0). Gebruikelijke waarde is 0.24.", ).execute() w = w.strip() - if w is "": - print(BColors.OKBLUE, f"Je hebt geen w gespecificeerd. Je invoer is leeg. ", BColors.ENDC) + if w == "": + print(BColors.OKBLUE, "Je hebt geen w gespecificeerd. Je invoer is leeg. ", BColors.ENDC) continue if not is_float(s=w): - print(BColors.OKBLUE, f"Je ingevoerde w is geen decimaal getal.", BColors.ENDC) + print(BColors.OKBLUE, "Je ingevoerde w is geen decimaal getal.", BColors.ENDC) continue w_float = float(w) if w_float > 1.0: - print(BColors.OKBLUE, f"Je ingevoerde w is groter dan 1.0. Het hoort een getal tussen 0.0 en 1.0 te zijn.", + print(BColors.OKBLUE, "Je ingevoerde w is groter dan 1.0. Het hoort een getal tussen 0.0 en 1.0 te zijn.", BColors.ENDC) continue if w_float <= 0.0: - print(BColors.OKBLUE, f"Je ingevoerde w is kleiner of gelijk aan 0.0. " - f"Het hoort een getal tussen 0.0 en 1.0 te zijn. ", BColors.ENDC) + print(BColors.OKBLUE, "Je ingevoerde w is kleiner of gelijk aan 0.0. " + "Het hoort een getal tussen 0.0 en 1.0 te zijn. ", BColors.ENDC) continue - w_is_valid = True + break _append_to_db(app_settings=app_settings, key='w', value=w_float) @@ -168,7 +163,7 @@ def _specify_w(app_settings: ApplicationSettings): def _specify_is_bovenrivierengebied(app_settings: ApplicationSettings): choices_list = ["Ja", "Nee"] choice = inquirer.select( - message=f"Is dit traject in het bovenrivierengebied?", choices=choices_list, default=choices_list[0]).execute() + message="Is dit traject in het bovenrivierengebied?", choices=choices_list, default=choices_list[0]).execute() choice_bool = False if choice == "Ja": choice_bool = True @@ -186,6 +181,7 @@ def added_traject_parameters(app_settings: ApplicationSettings) -> bool: _specify_signaleringswaarde(app_settings=app_settings) signaleringswaarde = _get_signaleringswaarde(app_settings=app_settings) + if "ondergrens" not in metadata_types: _specify_ondergrens(app_settings=app_settings, signaleringswaarde=signaleringswaarde) diff --git a/geoprob_pipe/cmd_app/parameter_input/added_input_parameters.py b/geoprob_pipe/cmd_app/parameter_input/added_input_parameters.py index f5b556a9..f3cb1899 100644 --- a/geoprob_pipe/cmd_app/parameter_input/added_input_parameters.py +++ b/geoprob_pipe/cmd_app/parameter_input/added_input_parameters.py @@ -1,16 +1,24 @@ from __future__ import annotations -from InquirerPy import inquirer + +import os import sqlite3 +import sys from datetime import datetime -from geoprob_pipe.cmd_app.parameter_input.expand_input_tables import run_expand_input_tables -from geoprob_pipe.cmd_app.parameter_input.initiate_input_excel_tables import initiate_input_excel_tables -from geoprob_pipe.cmd_app.parameter_input.input_parameter_figures import InputParameterFigures -from geoprob_pipe.cmd_app.parameter_input.export_input_parameter_excel import export_input_parameter_tables -from geoprob_pipe.cmd_app.parameter_input.input_parameter_tables import InputParameterTables from typing import TYPE_CHECKING, Optional -import os -import sys + +from InquirerPy import inquirer + from geoprob_pipe.utils.validation_messages import BColors + +from .expand_input_tables import run_expand_input_tables +from .export_input_parameter_excel import export_input_parameter_tables +from .initiate_input_excel_tables import initiate_input_excel_tables +from .input_parameter_figures import InputParameterFigures +from .input_parameter_tables import InputParameterTables +from .update_ruimtelijke_invoer import SpatialUpdateMenu +from geoprob_pipe.cmd_app.utils.batch_input import update_batch_metadata +from geoprob_pipe.utils.sql_contents import write_dfs_to_gpkg + if TYPE_CHECKING: from geoprob_pipe.cmd_app.cmd import ApplicationSettings @@ -25,8 +33,8 @@ def possibly_initiatie_input_tables_in_db(app_settings: ApplicationSettings): conn.close() # If already exist - if ("parameter_invoer" in tables_names and - "scenario_invoer" in tables_names and "fragility_values_invoer" in tables_names): + if ("data__excel_parameter_invoer" in tables_names and + "data__scenario_invoer" in tables_names and "data__fragility_values_invoer" in tables_names): # print(f"{BColors.UNDERLINE}Tables already exist in geopackage{BColors.ENDC}") return @@ -68,7 +76,7 @@ def inquire_if_input_figures_should_be_exported(app_settings: ApplicationSetting elif choice == choices_list[1]: pass # Just continue elif choice == choices_list[2]: - sys.exit(f"Applicatie is afgesloten.") + sys.exit("Applicatie is afgesloten.") else: raise ValueError @@ -109,10 +117,11 @@ def _export_expanded_input(app_settings: ApplicationSettings): # Copy template to workspace dst_dir = os.path.join( - app_settings.workspace_dir, + str(app_settings.workspace_dir), "exports", app_settings.datetime_stamp, - "parameter_input_process") + "parameter_input_process", + ) os.makedirs(dst_dir, exist_ok=True) dst_path = os.path.join(dst_dir, "expanded_parameter_input.xlsx") if os.path.exists(dst_path): @@ -130,76 +139,79 @@ def inquire_to_import_export_tables_and_figures_or_continue( ): # Determine options - choices_list = [] - if validity_raw_tables and validity_extended_tables: - choices_list.append("Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap") - else: - choices_list.append( - "Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap (n.v.t. -> invoer niet valide)") - if validity_raw_tables: - choices_list.append("Overzichtsfiguren van invoertabellen: Exporteren") - choices_list.extend([ - "Invoer tabellen: Importeren vanuit Excel", - "Invoer tabellen: Exporteren naar Excel", - "'Expanded' parameter invoer per uittredepunt: Exporteren", - "Toelichting per keuze optie", - "Applicatie afsluiten"]) - - # Provide user options - choice = inquirer.select( - message="Maak een keuze voor het gereedmaken van de invoertabellen.", - choices=choices_list, - default=choices_list[0], - ).execute() - - if choice == "Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap": - print(BColors.OKBLUE, f"✔ Parameter invoer afgerond.", BColors.ENDC) - return - - elif choice == "Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap (n.v.t. -> invoer niet valide)": - inquire_to_import_export_tables_and_figures_or_continue( - app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, - validity_extended_tables=validity_extended_tables) + while True: + choices_list = [] + if validity_raw_tables and validity_extended_tables: + choices_list.append("Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap") + else: + choices_list.append( + "Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap (n.v.t. -> invoer niet valide)") + if validity_raw_tables: + choices_list.append("Overzichtsfiguren van invoertabellen: Exporteren") + choices_list.extend([ + "Invoer tabellen: Importeren vanuit Excel", + "Invoer tabellen: Exporteren naar Excel", + "'Expanded' parameter invoer per uittredepunt: Exporteren", + "Ruimtelijke invoer: Update ingevoerde lagen", + "Toelichting per keuze optie", + "Applicatie afsluiten"]) + + # Provide user options + choice = inquirer.select( + message="Maak een keuze voor het gereedmaken van de invoertabellen.", + choices=choices_list, + default=choices_list[0], + ).execute() - elif choice == "Overzichtsfiguren van invoertabellen: Exporteren": - obj = InputParameterFigures.populate(app_settings=app_settings, tables=tables, export=True) - obj.run() - inquire_to_import_export_tables_and_figures_or_continue( - app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, - validity_extended_tables=validity_extended_tables) - - elif choice == "Invoer tabellen: Importeren vanuit Excel": - process_import_input(app_settings=app_settings) - - elif choice == "Invoer tabellen: Exporteren naar Excel": - process_export_input_of_db( - app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, - validity_extended_tables=validity_extended_tables) - - elif choice == "'Expanded' parameter invoer per uittredepunt: Exporteren": - _export_expanded_input(app_settings=app_settings) - inquire_to_import_export_tables_and_figures_or_continue( - app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, - validity_extended_tables=validity_extended_tables) - - elif choice == "Toelichting per keuze optie": - print(""" + match choice: + + case "Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap": + print(BColors.OKBLUE, "✔ Parameter invoer afgerond.", BColors.ENDC) + return + + case "Zijn de invoer tabellen zijn naar wens? Ga door naar volgende stap (n.v.t. -> invoer niet valide)": + continue + + case "Overzichtsfiguren van invoertabellen: Exporteren": + obj = InputParameterFigures.populate(app_settings=app_settings, tables=tables, export=True) + obj.run() + continue + + case "Invoer tabellen: Importeren vanuit Excel": + process_import_input(app_settings=app_settings) + continue + + case "Invoer tabellen: Exporteren naar Excel": + process_export_input_of_db( + app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, + validity_extended_tables=validity_extended_tables) + continue + + case "'Expanded' parameter invoer per uittredepunt: Exporteren": + _export_expanded_input(app_settings=app_settings) + continue + + case "Ruimtelijke invoer: Update ingevoerde lagen": + # Nieuw keuze menu voor de volgende: + menu = SpatialUpdateMenu(app_settings) + menu.inquire_spatial_update() + continue + + case "Toelichting per keuze optie": + print(""" Invoer tabellen zijn naar wens, ga door naar volgende stap -> Indien je deze keuzemogelijkheid krijgt zijn de invoertabellen valide en kun je door naar de volgende stap. Je zegt daarmee eveneens dat de invoertabellen naar wens zijn. Overzichtsfiguren van invoertabellen: Exporteren -> Deze interactive HTML-figuren geven je per parameter een snel visueel overzicht van de invoer in het GeoProb-Pipe-bestand. Invoer tabellen: Importeren vanuit Excel -> Importeer vanuit Excel de invoertabellen om ze te laten valideren, visualiseren en/of op te slaan in het GeoProb-Pipe-bestand. Invoer tabellen: Exporteren naar Excel -> Exporteer vanuit het GeoProb-Pipe-bestand de invoertabellen om ze in Excel te bekijken en/of verder aan te vullen. 'Expanded' parameter invoer: Exporteren -> Exporteert een Excel met de parameter invoer 'expanded' naar per uittredepunt en per scenario. 'Expanded' vanuit de invoer op verschillende niveau's. - """) - inquire_to_import_export_tables_and_figures_or_continue( - app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, - validity_extended_tables=validity_extended_tables) +Ruimtelijke invoer: Update ingevoerde lagen -> Open een keuze menu om de lagen die zijn ingevoerd voor ruimtelijke koppeling van parameters te vervangen, verwijderen of toe te voegen. + """) + continue - elif choice == "Applicatie afsluiten": - sys.exit(f"Applicatie is afgesloten.") + case "Applicatie afsluiten": + sys.exit("Applicatie is afgesloten.") - else: - raise ValueError def export_input_tables_of_db(app_settings: ApplicationSettings, tables: InputParameterTables): @@ -207,10 +219,7 @@ def export_input_tables_of_db(app_settings: ApplicationSettings, tables: InputPa def import_input_tables(geopackage_filepath: str) -> InputParameterTables: - - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: + while True: filepath: str = inquirer.text( message="Specificeer het volledige bestandspad naar het input parameters Excel.", ).execute() @@ -222,12 +231,12 @@ def import_input_tables(geopackage_filepath: str) -> InputParameterTables: f"{os.path.basename(filepath)} eindigt niet op deze extensie.", BColors.ENDC) continue if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) + print(BColors.WARNING, "Het opgegeven bestandspad bestaat niet.", BColors.ENDC) continue - filepath_is_valid = True + break - tables = InputParameterTables(path_to_excel=filepath, geopackage_filepath=geopackage_filepath) + tables = InputParameterTables(path_to_excel=filepath, geopackage_filepath=geopackage_filepath) # type:ignore print(f"{BColors.UNDERLINE}Tabellen zijn nu geïmporteerd.{BColors.ENDC}") return tables @@ -243,13 +252,20 @@ def inquire_to_store_input_tables_to_db( ).execute() if choice == "Ja": + conn = sqlite3.connect(app_settings.geopackage_filepath) - tables.df_scenario_invoer.to_sql("scenario_invoer", conn, if_exists="replace", index=False) - tables.df_parameter_invoer.to_sql("parameter_invoer", conn, if_exists="replace", index=False) - tables.df_fragility_values_invoer.to_sql( - "fragility_values_invoer", conn, if_exists="replace", index=False) - tables.df_correlatie_invoer.to_sql("correlatie_invoer", conn, if_exists="replace", index=False) + + table_dict = { + "data__scenario_invoer": tables.df_scenario_invoer, + "data__excel_parameter_invoer": tables.df_parameter_invoer, + "data__fragility_values_invoer": tables.df_fragility_values_invoer, + "data__correlatie_invoer": tables.df_correlatie_invoer + } + + write_dfs_to_gpkg(conn, tables=table_dict) + conn.close() + print(f"{BColors.UNDERLINE}Tabellen zijn nu opgeslagen in het GeoProb-Pipe-file.{BColors.ENDC}") elif choice == "Nee": @@ -267,7 +283,7 @@ def process_input_exist_in_db(app_settings: ApplicationSettings): validity_raw_tables = validate_raw_input_tables(app_settings=app_settings, tables=tables) # Validate expanded tables - validity_extended_tables: Optional[bool] = None + validity_extended_tables: bool = False if validity_raw_tables: validity_extended_tables = validate_expanded_input_tables(app_settings=app_settings, tables=tables) @@ -282,10 +298,6 @@ def process_export_input_of_db( app_settings: ApplicationSettings, tables: InputParameterTables, validity_raw_tables: bool, validity_extended_tables: bool): export_input_tables_of_db(app_settings=app_settings, tables=tables) - inquire_to_import_export_tables_and_figures_or_continue( - app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, - validity_extended_tables=validity_extended_tables) - # -> Redirects also to new process loop def process_import_input(app_settings: ApplicationSettings): @@ -296,21 +308,23 @@ def process_import_input(app_settings: ApplicationSettings): validity_raw_tables = validate_raw_input_tables(app_settings=app_settings, tables=tables) # Ask to export overview pictures - if validity_raw_tables: inquire_if_input_figures_should_be_exported(app_settings=app_settings, tables=tables) + if validity_raw_tables: + inquire_if_input_figures_should_be_exported(app_settings=app_settings, tables=tables) # Validate expanded tables - validity_extended_tables: Optional[bool] = None + validity_extended_tables: bool = False if validity_raw_tables: validity_extended_tables = validate_expanded_input_tables(app_settings=app_settings, tables=tables) + if not validity_extended_tables: + return + # Provide user with follow-up options inquire_to_store_input_tables_to_db(app_settings=app_settings, tables=tables) - inquire_to_import_export_tables_and_figures_or_continue( - app_settings=app_settings, tables=tables, validity_raw_tables=validity_raw_tables, - validity_extended_tables=validity_extended_tables) - # -> Redirects also to new process loop def added_input_parameter_data(app_settings: ApplicationSettings) -> bool: + # Batch inquiry flag is set to False if input is successful. + update_batch_metadata(app_settings=app_settings, value=False) process_input_exist_in_db(app_settings=app_settings) return True diff --git a/geoprob_pipe/cmd_app/parameter_input/expand_input_tables.py b/geoprob_pipe/cmd_app/parameter_input/expand_input_tables.py index a16d32c5..d48a87e4 100644 --- a/geoprob_pipe/cmd_app/parameter_input/expand_input_tables.py +++ b/geoprob_pipe/cmd_app/parameter_input/expand_input_tables.py @@ -1,5 +1,5 @@ from typing import Dict, List, Optional -from pandas import DataFrame, isna, notna, concat, read_sql, read_csv, set_option +from pandas import DataFrame, Series, isna, notna, concat, read_sql, read_csv, set_option import sqlite3 import numpy as np import os @@ -7,7 +7,9 @@ from geopandas import GeoDataFrame, read_file from geoprob_pipe.calculations.systems.mappers.initial_input import INITIAL_INPUT_MAPPER from geoprob_pipe.cmd_app.parameter_input.input_parameter_tables import InputParameterTables +from geoprob_pipe.utils.sql_contents import write_dfs_to_gpkg from probabilistic_library import FragilityValue +import copy def _combine_parameter_invoer_sources(tables: InputParameterTables) -> DataFrame: @@ -33,7 +35,7 @@ def _gather_hrd_frag_line_from_geopackage(ref: str, geopackage_filepath: str): # Read database conn = sqlite3.connect(geopackage_filepath) df_frag_line: DataFrame = read_sql( - sql=f"SELECT * FROM fragility_values_invoer_hrd WHERE fragility_values_ref = '{ref}' AND kans < 1.0;", + sql=f"SELECT * FROM data__fragility_values_invoer_hrd WHERE fragility_values_ref = '{ref}' AND kans < 1.0;", con=conn) conn.close() @@ -61,7 +63,7 @@ def _gather_hrd_frag_line_from_geopackage(ref: str, geopackage_filepath: str): def _gather_other_freq_line(ref: str, tables: InputParameterTables): """ Gather a freq line that is not from the HRD-database, and not from a .csv-file. It originates from the Excel, - either directly from the Excel, or imported into the 'fragility_values_invoer_hrd'-database table. + either directly from the Excel, or imported into the 'data__fragility_values_invoer_hrd'-database table. :param ref: :param tables: @@ -222,7 +224,7 @@ def _construct_df_identifiers(geopackage_filepath: str, tables: InputParameterTa The identifiers table is a table with unique rows (unique uittredepunt, vak and scenario combo). It is used as a base to explode to input per scenario and uittredepunt. """ - gdf_uittredepunten: GeoDataFrame = read_file(geopackage_filepath, layer="uittredepunten") + gdf_uittredepunten: GeoDataFrame = read_file(geopackage_filepath, layer="geom__uittredepunten") df_identifiers: DataFrame = gdf_uittredepunten[["uittredepunt_id", "vak_id"]] df_scenario_invoer: DataFrame = tables.df_scenario_invoer df_identifiers = df_identifiers.merge(df_scenario_invoer, left_on="vak_id", right_on="vak_id") @@ -235,7 +237,7 @@ def _gather_required_input_parameters(geopackage_filepath: str) -> List[str]: conn = sqlite3.connect(geopackage_filepath) cursor = conn.cursor() cursor.execute(""" - SELECT geoprob_pipe_metadata."values" + SELECT geoprob_pipe_metadata.metadata_value FROM geoprob_pipe_metadata WHERE metadata_type='geohydrologisch_model'; """) @@ -267,82 +269,134 @@ def _gather_required_input_parameters(geopackage_filepath: str) -> List[str]: # return ["mv_exit", "gamma_sat_deklaag"] # TODO # return ["gamma_sat_deklaag"] # TODO - def _expand( - df_parameter_invoer_combined: DataFrame, df_identifiers: DataFrame, geopackage_filepath: str + df_parameter_invoer_combined: DataFrame, + df_identifiers: DataFrame, + geopackage_filepath: str, ) -> Dict[str, DataFrame]: - # Add parameter invoer: op uittredepunten niveau - required_input_parameters = _gather_required_input_parameters(geopackage_filepath=geopackage_filepath) + + def merge_into_df(df: DataFrame, df_gather: DataFrame, on_cols: list[str]): + """Veilige merge + combine_first""" + if df_gather.empty: + return df + + df = df.merge( + df_gather, + on=on_cols, + how="left", + suffixes=("", "_new"), + ) + + df["parameter_input"] = df["parameter_input"].combine_first( + df["parameter_input_new"] + ) + df.drop(columns=["parameter_input_new"], inplace=True) + + return df + + required_input_parameters = _gather_required_input_parameters( + geopackage_filepath=geopackage_filepath + ) + collection_of_dfs: Dict[str, DataFrame] = {} for parameter_name in required_input_parameters: - # Gather and merge input on uittredepunten / scenario-niveau - df_gather = df_parameter_invoer_combined[ - (df_parameter_invoer_combined['parameter'] == parameter_name) & - (df_parameter_invoer_combined['scope'] == 'uittredepunt') & - (df_parameter_invoer_combined['ondergrondscenario_naam'].notna())] - df_gather = df_gather[["scope_referentie", "ondergrondscenario_naam", "parameter_input"]] - df_gather = df_gather.rename(columns={ - "scope_referentie": "uittredepunt_id", - "ondergrondscenario_naam": "naam"}) - df = df_identifiers.copy(deep=True) - df["parameter_input"] = np.nan - df['parameter_input'] = df['parameter_input'].combine_first( - df_identifiers.copy(deep=True).merge(df_gather, on=["uittredepunt_id", "naam"], how="left")[ - 'parameter_input']) - - # Uittredepunt niveau - df_gather = df_parameter_invoer_combined[ - (df_parameter_invoer_combined['parameter'] == parameter_name) & - (df_parameter_invoer_combined['scope'] == 'uittredepunt') & - (df_parameter_invoer_combined['ondergrondscenario_naam']).isna()] - df_gather = df_gather[["scope_referentie", "parameter_input"]] - df_gather = df_gather.rename(columns={"scope_referentie": "uittredepunt_id"}) - - df['parameter_input'] = df['parameter_input'].combine_first( - df_identifiers.copy(deep=True).merge(df_gather, on=["uittredepunt_id"], how="left")['parameter_input']) - - # Vak / scenario niveau - df_gather = df_parameter_invoer_combined[ - (df_parameter_invoer_combined['parameter'] == parameter_name) & - (df_parameter_invoer_combined['scope'] == 'vak') & - (df_parameter_invoer_combined['ondergrondscenario_naam'].notna())] - df_gather = df_gather[["scope_referentie", "ondergrondscenario_naam", "parameter_input"]] - df_gather = df_gather.rename(columns={ - "scope_referentie": "vak_id", - "ondergrondscenario_naam": "naam"}) - df['parameter_input'] = df['parameter_input'].combine_first( - df_identifiers.copy(deep=True).merge(df_gather, on=["vak_id", "naam"], how="left")['parameter_input']) - - # Vak niveau - df_gather = df_parameter_invoer_combined[ - (df_parameter_invoer_combined['parameter'] == parameter_name) & - (df_parameter_invoer_combined['scope'] == 'vak') & - (df_parameter_invoer_combined['ondergrondscenario_naam'].isna())] - df_gather = df_gather[["scope_referentie", "parameter_input"]] - df_gather = df_gather.rename(columns={"scope_referentie": "vak_id"}) - df['parameter_input'] = df['parameter_input'].combine_first( - df_identifiers.copy(deep=True).merge(df_gather, on=["vak_id"], how="left")['parameter_input']) - - # Traject niveau - df_gather = df_parameter_invoer_combined[ - (df_parameter_invoer_combined['parameter'] == parameter_name) & - (df_parameter_invoer_combined['scope'] == 'traject')] - if df_gather.__len__() >= 1: - traject_value = df_gather['parameter_input'].values[0] - df['parameter_input'] = df['parameter_input'].apply(lambda x: traject_value if isna(x) else x) - - # GIS spatial joins - df_gather = df_parameter_invoer_combined[ - (df_parameter_invoer_combined['parameter'] == parameter_name) & - (df_parameter_invoer_combined['scope'] == 'gis_uittredepunt')].copy(deep=True) - df_gather = df_gather[["scope_referentie", "parameter_input"]] - df_gather = df_gather.rename(columns={"scope_referentie": "uittredepunt_id"}) - df['parameter_input'] = df['parameter_input'].combine_first( - df_identifiers.copy(deep=True).merge(df_gather, on=["uittredepunt_id"], how="left")['parameter_input']) - - # Add to collection + df = df_identifiers.copy() + df["parameter_input"] = Series([None] * len(df), dtype="object") + + # 1. excel uittrdepunt en scenario + df_gather = df_parameter_invoer_combined.loc[ + (df_parameter_invoer_combined["parameter"] == parameter_name) + & (df_parameter_invoer_combined["scope"] == "uittredepunt") + & (df_parameter_invoer_combined["ondergrondscenario_naam"].notna()), + ["scope_referentie", "ondergrondscenario_naam", "parameter_input"], + ].rename( + columns={ + "scope_referentie": "uittredepunt_id", + "ondergrondscenario_naam": "naam", + } + ).drop_duplicates(["uittredepunt_id", "naam"]) + + df = merge_into_df(df, df_gather, ["uittredepunt_id", "naam"]) + + # 2. GIS uittredepunt en scenario + df_gather = df_parameter_invoer_combined.loc[ + (df_parameter_invoer_combined["parameter"] == parameter_name) + & (df_parameter_invoer_combined["scope"] == "gis_uittredepunt") + & (df_parameter_invoer_combined["ondergrondscenario_naam"].notna()), + ["scope_referentie", "ondergrondscenario_naam", "parameter_input"], + ].rename( + columns={ + "scope_referentie": "uittredepunt_id", + "ondergrondscenario_naam": "naam", + } + ).drop_duplicates(["uittredepunt_id", "naam"]) + + df = merge_into_df(df, df_gather, ["uittredepunt_id", "naam"]) + + # 3. excel uittrdepunt + df_gather = df_parameter_invoer_combined.loc[ + (df_parameter_invoer_combined["parameter"] == parameter_name) + & (df_parameter_invoer_combined["scope"] == "uittredepunt") + & (df_parameter_invoer_combined["ondergrondscenario_naam"].isna()) + & (df_parameter_invoer_combined["parameter_input"] != {}), + ["scope_referentie", "parameter_input"], + ].rename(columns={"scope_referentie": "uittredepunt_id"}) \ + .drop_duplicates(["uittredepunt_id"]) + + df = merge_into_df(df, df_gather, ["uittredepunt_id"]) + + # 4. GIS uittredepunt + df_gather = df_parameter_invoer_combined.loc[ + (df_parameter_invoer_combined["parameter"] == parameter_name) + & (df_parameter_invoer_combined["scope"] == "gis_uittredepunt") + & (df_parameter_invoer_combined["parameter_input"] != {}), + ["scope_referentie", "parameter_input"], + ].rename(columns={"scope_referentie": "uittredepunt_id"}) \ + .drop_duplicates(["uittredepunt_id"]) + + df = merge_into_df(df, df_gather, ["uittredepunt_id"]) + + # 5. excel vak en scenario + df_gather = df_parameter_invoer_combined.loc[ + (df_parameter_invoer_combined["parameter"] == parameter_name) + & (df_parameter_invoer_combined["scope"] == "vak") + & (df_parameter_invoer_combined["ondergrondscenario_naam"].notna()), + ["scope_referentie", "ondergrondscenario_naam", "parameter_input"], + ].rename( + columns={ + "scope_referentie": "vak_id", + "ondergrondscenario_naam": "naam", + } + ).drop_duplicates(["vak_id", "naam"]) + + df = merge_into_df(df, df_gather, ["vak_id", "naam"]) + + # 6. excel vak + df_gather = df_parameter_invoer_combined.loc[ + (df_parameter_invoer_combined["parameter"] == parameter_name) + & (df_parameter_invoer_combined["scope"] == "vak") + & (df_parameter_invoer_combined["ondergrondscenario_naam"].isna()), + ["scope_referentie", "parameter_input"], + ].rename(columns={"scope_referentie": "vak_id"}) \ + .drop_duplicates(["vak_id"]) + + df = merge_into_df(df, df_gather, ["vak_id"]) + + # 7. excel traject + df_gather = df_parameter_invoer_combined.loc[ + (df_parameter_invoer_combined["parameter"] == parameter_name) + & (df_parameter_invoer_combined["scope"] == "traject") + ] + if not df_gather.empty: + value = df_gather.iloc[0]["parameter_input"] + mask_na = df["parameter_input"].isna() + + df.loc[mask_na, "parameter_input"] = [ + copy.deepcopy(value) for _ in range(mask_na.sum()) + ] + collection_of_dfs[parameter_name] = df return collection_of_dfs @@ -386,5 +440,5 @@ def run_expand_input_tables( df_parameter_invoer_combined=df_parameter_invoer_combined3, df_identifiers=df_identifiers, geopackage_filepath=geopackage_filepath) - + return _concat_collection(collection=collection) diff --git a/geoprob_pipe/cmd_app/parameter_input/export_input_parameter_excel.py b/geoprob_pipe/cmd_app/parameter_input/export_input_parameter_excel.py index 600ffc6e..04e39f30 100644 --- a/geoprob_pipe/cmd_app/parameter_input/export_input_parameter_excel.py +++ b/geoprob_pipe/cmd_app/parameter_input/export_input_parameter_excel.py @@ -16,7 +16,7 @@ def join_vak_naam(df: DataFrame, app_settings: ApplicationSettings, ) -> DataFrame: - gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="vakindeling") + gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__vakindeling") gdf_vakindeling = gdf_vakindeling.rename(columns={"naam": "vak_naam", "id": "vak_id"}) df = df.merge(gdf_vakindeling[["vak_naam", "vak_id"]], left_on="vak_id", right_on="vak_id") return df[["vak_id", "vak_naam", "naam", "kans"]] @@ -46,7 +46,7 @@ def export_input_parameter_tables(app_settings: ApplicationSettings, tables: Inp conn = sqlite3.connect(app_settings.geopackage_filepath) cursor = conn.cursor() cursor.execute(""" - SELECT geoprob_pipe_metadata."values" + SELECT geoprob_pipe_metadata.metadata_value FROM geoprob_pipe_metadata WHERE metadata_type='geohydrologisch_model'; """) diff --git a/geoprob_pipe/cmd_app/parameter_input/initiate_input_excel_tables.py b/geoprob_pipe/cmd_app/parameter_input/initiate_input_excel_tables.py index ce0266e5..3eff152e 100644 --- a/geoprob_pipe/cmd_app/parameter_input/initiate_input_excel_tables.py +++ b/geoprob_pipe/cmd_app/parameter_input/initiate_input_excel_tables.py @@ -13,7 +13,7 @@ def push_scenario_invoer_tabel(app_settings: ApplicationSettings): # Genereer tabel input (o.b.v. vakindeling) - gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="vakindeling") + gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__vakindeling") gdf_vakindeling = gdf_vakindeling.sort_values(by=["id"]) df_scenarios = gdf_vakindeling[["id"]].copy() df_scenarios.loc[:, 'naam'] = "scenario1" @@ -24,12 +24,12 @@ def push_scenario_invoer_tabel(app_settings: ApplicationSettings): # Push to geopackage conn = sqlite3.connect(app_settings.geopackage_filepath) - df_scenarios.to_sql("scenario_invoer", conn, if_exists="replace", index=False) + df_scenarios.to_sql("data__scenario_invoer", conn, if_exists="replace", index=False) conn.close() def _check_to_exclude_buitenwaterstand(app_settings: ApplicationSettings, df_parameter_invoer: DataFrame): - gdf_uittredepunten: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") + gdf_uittredepunten: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__uittredepunten") if "hrd_name" in gdf_uittredepunten.columns: return df_parameter_invoer[df_parameter_invoer["name"] != "buitenwaterstand"] return df_parameter_invoer @@ -47,7 +47,7 @@ def push_parameter_invoer_tabel(app_settings: ApplicationSettings): # Exclude parameters that must be geographically added also df_parameter_invoer = df_parameter_invoer[ - ~df_parameter_invoer["name"].isin(["L_intrede", "L_but", "L_bit", "polderpeil", "mv_exit"])] + ~df_parameter_invoer["name"].isin(["L_intrede", "L_but", "L_bit", "mv_exit"])] df_parameter_invoer = _check_to_exclude_buitenwaterstand( app_settings=app_settings, df_parameter_invoer=df_parameter_invoer) @@ -75,7 +75,7 @@ def push_parameter_invoer_tabel(app_settings: ApplicationSettings): # Push to geopackage conn = sqlite3.connect(app_settings.geopackage_filepath) - df_parameter_invoer.to_sql("parameter_invoer", conn, if_exists="replace", index=False) + df_parameter_invoer.to_sql("data__excel_parameter_invoer", conn, if_exists="replace", index=False) conn.close() @@ -92,7 +92,7 @@ def push_fragility_values_invoer_tabel(app_settings: ApplicationSettings): # Push to geopackage conn = sqlite3.connect(app_settings.geopackage_filepath) - df_fragility_values_invoer.to_sql("fragility_values_invoer", conn, if_exists="replace", index=False) + df_fragility_values_invoer.to_sql("data__fragility_values_invoer", conn, if_exists="replace", index=False) conn.close() @@ -102,7 +102,7 @@ def push_fragility_values_invoer_tabel(app_settings: ApplicationSettings): def push_correlatie_invoer_tabel(app_settings: ApplicationSettings): # Push to geopackage conn = sqlite3.connect(app_settings.geopackage_filepath) - DF_EMPTY_CORRELATIE_INVOER.to_sql("correlatie_invoer", conn, if_exists="replace", index=False) + DF_EMPTY_CORRELATIE_INVOER.to_sql("data__correlatie_invoer", conn, if_exists="replace", index=False) conn.close() diff --git a/geoprob_pipe/cmd_app/parameter_input/input_parameter_figures.py b/geoprob_pipe/cmd_app/parameter_input/input_parameter_figures.py index 547e132a..796e82f7 100644 --- a/geoprob_pipe/cmd_app/parameter_input/input_parameter_figures.py +++ b/geoprob_pipe/cmd_app/parameter_input/input_parameter_figures.py @@ -32,9 +32,9 @@ def __init__(self): self.export_sub_dir: str = "parameter_input_process" # Placeholders - self.df_parameter_invoer: Optional[DataFrame] = None + self.df_parameter_invoer: DataFrame = DataFrame() self.dict_vakindeling: Dict[int, Dict] = {} - self.traject_length: Optional[float] = None + self.traject_length: float = 0.0 self.x_min = 0 self.x_max = 0 # Will be set self.y_min = 0 @@ -86,17 +86,17 @@ def _gather_data(self): # Lengte traject gpkg_file_path = self.app_settings.geopackage_filepath - gdf: GeoDataFrame = read_file(gpkg_file_path, layer="dijktraject") + gdf: GeoDataFrame = read_file(gpkg_file_path, layer="geom__dijktraject") assert gdf.__len__() == 1 self.traject_length = gdf.iloc[0].geometry.length self.x_max = self.traject_length # Vakindeling dictionary - gdf_vakindeling: GeoDataFrame = read_file(self.app_settings.geopackage_filepath, layer="vakindeling") + gdf_vakindeling: GeoDataFrame = read_file(self.app_settings.geopackage_filepath, layer="geom__vakindeling") self.dict_vakindeling = gdf_vakindeling.set_index('id').to_dict(orient='index') # Uittredepunten dictionary - gdf_uittredepunten: GeoDataFrame = read_file(self.app_settings.geopackage_filepath, layer="uittredepunten") + gdf_uittredepunten: GeoDataFrame = read_file(self.app_settings.geopackage_filepath, layer="geom__uittredepunten") self.dict_uittredepunten = gdf_uittredepunten.set_index('uittredepunt_id').to_dict(orient='index') @staticmethod @@ -121,7 +121,7 @@ def _get_display_values_from_row(row) -> Tuple[float, Optional[float], Optional[ elif not np.isnan(variation): deviation_value = mean_value * variation else: - ValueError(f"Should have either a variation or deviation. Or maybe a bug and contact the developer.") + ValueError("Should have either a variation or deviation. Or maybe a bug and contact the developer.") # Log normal if row['distribution_type'] == 'log_normal': @@ -152,7 +152,7 @@ def _get_display_values_from_df( kar5pr_values = [] kar95pr_values = [] - for index, row in df.iterrows(): + for _, row in df.iterrows(): mean_value = row['mean'] mean_values.append(mean_value) if row['distribution_type'] == 'deterministic': @@ -165,7 +165,7 @@ def _get_display_values_from_df( elif not np.isnan(row['variation']): deviation_value = mean_value * row['variation'] else: - ValueError(f"Should have either a variation or deviation. Or maybe a bug and contact the developer.") + ValueError("Should have either a variation or deviation. Or maybe a bug and contact the developer.") if row['distribution_type'] == 'log_normal': kar5pr_values.append(calc_kar_waarde_lognormal( mean=mean_value, sd=deviation_value, percentiel=0.05)) # TODO: Shift bepalen @@ -185,7 +185,7 @@ def _add_geospatial_level_data(self, fig: go.Figure, parameter_name: str) -> go. (self.df_gis_join_parameter_invoer['scope'] == 'uittredepunt')] show_legend_item_mean = True - for index, row in df_filter.iterrows(): + for _, row in df_filter.iterrows(): # Skip cdf curve if row['distribution_type'] == 'cdf_curve': diff --git a/geoprob_pipe/cmd_app/parameter_input/input_parameter_tables.py b/geoprob_pipe/cmd_app/parameter_input/input_parameter_tables.py index 15062cfa..797bb907 100644 --- a/geoprob_pipe/cmd_app/parameter_input/input_parameter_tables.py +++ b/geoprob_pipe/cmd_app/parameter_input/input_parameter_tables.py @@ -17,7 +17,7 @@ def _load_df_correlatie_invoer_from_geopackage(geopackage_filepath: str) -> Data # Check if table exists in geopackage (older versions don't have this) conn = sqlite3.connect(geopackage_filepath) cursor = conn.cursor() - cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='correlatie_invoer';") + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='correlatie_invoer';") bool_table_exists = cursor.fetchone() is not None # Return empty if not exists @@ -25,7 +25,7 @@ def _load_df_correlatie_invoer_from_geopackage(geopackage_filepath: str) -> Data return DF_EMPTY_CORRELATIE_INVOER # Return db table if exists - df_correlatie_invoer = read_sql("SELECT * FROM correlatie_invoer;", conn) + df_correlatie_invoer = read_sql("SELECT * FROM data__correlatie_invoer;", conn) conn.close() return df_correlatie_invoer @@ -70,10 +70,10 @@ class InputParameterTables: def __init__(self, geopackage_filepath: str, path_to_excel: Optional[str] = None): # Placeholders - self.df_scenario_invoer: Optional[DataFrame] = None - self.df_parameter_invoer: Optional[DataFrame] = None - self.df_fragility_values_invoer: Optional[DataFrame] = None - self.df_correlatie_invoer: Optional[DataFrame] = None + # self.df_scenario_invoer: Optional[DataFrame] = None + # self.df_parameter_invoer: Optional[DataFrame] = None + # self.df_fragility_values_invoer: Optional[DataFrame] = None + # self.df_correlatie_invoer: Optional[DataFrame] = None if path_to_excel is not None: self._load_data_from_excel(path_to_excel=path_to_excel, geopackage_filepath=geopackage_filepath) @@ -82,11 +82,11 @@ def __init__(self, geopackage_filepath: str, path_to_excel: Optional[str] = None def _load_data_from_geopackage(self, geopackage_filepath: str): conn = sqlite3.connect(geopackage_filepath) - df_scenario_invoer = read_sql("SELECT * FROM scenario_invoer;", conn) + df_scenario_invoer = read_sql("SELECT * FROM data__scenario_invoer;", conn) self.df_scenario_invoer = df_scenario_invoer[["vak_id", "naam", "kans"]] - self.df_parameter_invoer = read_sql("SELECT * FROM parameter_invoer;", conn) - self.df_gis_join_parameter_invoer = read_sql("SELECT * FROM gis_join_parameter_invoer;", conn) - self.df_fragility_values_invoer = read_sql("SELECT * FROM fragility_values_invoer;", conn) + self.df_parameter_invoer = read_sql("SELECT * FROM data__excel_parameter_invoer;", conn) + self.df_gis_join_parameter_invoer = read_sql("SELECT * FROM data__gis_parameter_invoer;", conn) + self.df_fragility_values_invoer = read_sql("SELECT * FROM data__fragility_values_invoer;", conn) self.df_correlatie_invoer = _load_df_correlatie_invoer_from_geopackage(geopackage_filepath=geopackage_filepath) conn.close() @@ -94,11 +94,13 @@ def _load_data_from_excel(self, path_to_excel: str, geopackage_filepath: str): self.df_scenario_invoer = read_excel(path_to_excel, sheet_name="Scenario invoer", header=2) self.df_parameter_invoer = read_excel(path_to_excel, sheet_name="Parameter invoer", header=3) conn = sqlite3.connect(geopackage_filepath) - self.df_gis_join_parameter_invoer = read_sql("SELECT * FROM gis_join_parameter_invoer;", conn) + self.df_gis_join_parameter_invoer = read_sql("SELECT * FROM data__gis_parameter_invoer;", conn) conn.close() self.df_fragility_values_invoer = read_excel(path_to_excel, sheet_name="Fragility values", header=3) self.df_correlatie_invoer = read_excel(path_to_excel, sheet_name="Correlatie invoer", header=3) def validate_and_report(self, app_settings: ApplicationSettings) -> bool: - if not _validate_df_parameter_invoer(df=self.df_parameter_invoer, app_settings=app_settings): return False + if not _validate_df_parameter_invoer(df=self.df_parameter_invoer, app_settings=app_settings): + return False + return True diff --git a/geoprob_pipe/cmd_app/parameter_input/update_ruimtelijke_invoer.py b/geoprob_pipe/cmd_app/parameter_input/update_ruimtelijke_invoer.py new file mode 100644 index 00000000..688e09e0 --- /dev/null +++ b/geoprob_pipe/cmd_app/parameter_input/update_ruimtelijke_invoer.py @@ -0,0 +1,521 @@ +from __future__ import annotations + +import sqlite3 +import sys +from typing import TYPE_CHECKING + +import fiona +from InquirerPy.prompts.input import InputPrompt +from InquirerPy.prompts.list import ListPrompt + +from geoprob_pipe.cmd_app.spatial_layers import valid_parameter_list +from geoprob_pipe.utils.validation_messages import BColors +from geoprob_pipe.cmd_app.utils.batch_input import update_batch_metadata + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +class SpatialUpdateMenu: + restart_flag: bool = False + + def __init__( + self, + app_settings: ApplicationSettings, + ) -> None: + """ + Class voor alle submenus onder het aanpassen vaan de ruimtelijke invoer. + + :param app_settings: Object met de instellingen van de applicatie + """ + self.app_settings = app_settings + + def inquire_spatial_update( + self, + ) -> None: + """ + Keuzemenu voor het updaten van de ruimtelijke invoer van de .gpkg. + + :param app_settings: Object met de instellingen van de applicatie + """ + + + while True: + choices_list = [ + "Vervang ingevoerde lagen", + "Voeg parameters toe voor ruimtelijke invoer", + "Verwijder parameters voor ruimtelijke invoer", + "Voeg scenarios toe voor ruimtelijke invoer", + "Verwijder scenarios voor ruimtelijke invoer", + ] + + if self.restart_flag: + choices_list.append("Applicatie afsluiten na aanpassingen") + else: + choices_list.append("Ga terug naar het parameter invoer menu") + + choice = ListPrompt( + message="Maak een keuze voor het aanpassen van de ruimtelijke invoer.", + choices=choices_list, + default=choices_list[0], + ).execute() + + match choice: + case "Vervang ingevoerde lagen": + self.inquire_replace_layers() + continue + + case "Voeg parameters toe voor ruimtelijke invoer": + self.add_parameters() + continue + + case "Verwijder parameters voor ruimtelijke invoer": + self.remove_parameters() + continue + + case "Voeg scenarios toe voor ruimtelijke invoer": + self.add_scenarios() + continue + + case "Verwijder scenarios voor ruimtelijke invoer": + self.remove_scenarios() + continue + + case "Applicatie afsluiten na aanpassingen": + sys.exit("Applicatie is afgesloten.") + + case "Ga terug naar het parameter invoer menu": + return + + def inquire_replace_layers(self) -> None: + """ + Verwijder tabellen die opnieuw moeten worden ingeladen om de geopackage te + updaten. + """ + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + + # Haal ruimtelijke parameters op vanuit de metadata + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_parameters",), + ) + parameters: list[str] = cursor.fetchone()[0].split(", ") + + conn.commit() + conn.close() + + valid_list = valid_parameter_list(self.app_settings) + + while True: + parameter_input: str = InputPrompt( + message=f""" +De volgende parameters zijn nu aangemerkt voor ruimtelijke invoer: +{", ".join(parameters)} +Welke parameters wil je updaten? Geef deze met comma's gescheiden op. +De tabellen worden verwijderd uit de geopackage zodat deze opnieuw worden ingelezen. +Als er niets wordt opgegeven wordt je terug gestuurd naar het keuzemenu. +""" + ).execute() + + if parameter_input == "": + print( + BColors.OKBLUE, + "Geen tabellen verwijderd.", + BColors.ENDC, + ) + return + + valid_input = all( + [ + param + for param in parameter_input.split(",") + if param in valid_list + ] + ) + if not valid_input: + print( + BColors.OKBLUE, + f"Geen parameters toegevoegd. '{parameter_input}' bevat een ongeldige parameter.", + BColors.ENDC, + ) + continue + + break + + # Verwijder uit geopackage + layers: list[str] = fiona.listlayers( + self.app_settings.geopackage_filepath + ) + table_list = [ + layer + for layer in layers + if layer.split("__")[0] in parameter_input + ] + self.remove_tables(table_list) + self.completed_update() + + def add_parameters(self) -> None: + """ + Voeg extra parameters toe voor de ruimtelijke invoer. De aanpassingen + kunnen pas verwerkt worden nadat de applicatie opnieuw is opgestart. + """ + + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + + # Haal ruimtelijke parameters op vanuit de metadata + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_parameters",), + ) + current_parameters: list[str] = cursor.fetchone()[0].split(", ") + + conn.commit() + conn.close() + + # Haal lijst met geldige parameters op. + valid_list = valid_parameter_list(self.app_settings) + + # Vraag gebruiker voor toevoegingen. + while True: + parameter_input: str = InputPrompt( + message=f""" +De volgende parameters zijn nu aangemerkt voor ruimtelijke invoer: +{", ".join(current_parameters)} +De volgende parameters kunnen worden toegevoegd: +{[param for param in valid_list if param not in current_parameters]} +Welke parameters wil je hieraan toevoegen? Geef deze met comma's gescheiden op. +Als er niets wordt opgegeven wordt je terug gestuurd naar het keuzemenu. +""" + ).execute() + + if parameter_input == "": + print( + BColors.OKBLUE, + "Geen nieuwe parameters toegevoegd.", + BColors.ENDC, + ) + return + + valid_input = all( + [ + param + for param in parameter_input.split(",") + if param in valid_list + ] + ) + + if not valid_input: + print( + BColors.OKBLUE, + f"Geen parameters toegevoegd. '{parameter_input}' bevat een ongeldige parameter.", + BColors.ENDC, + ) + continue + + break + + # Update metadata + parameters = current_parameters + parameter_input.split(",") + parameters = list(set(parameters)) # Verwijder dubbele waardes + parameters = ", ".join(parameters) + self.update_metadata_parameters(parameters) + + self.completed_update() + + def remove_parameters(self) -> None: + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + + # Haal ruimtelijke parameters op vanuit de metadata + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_parameters",), + ) + parameters: list[str] = cursor.fetchone()[0].split(", ") + + conn.commit() + conn.close() + + valid_list = valid_parameter_list(self.app_settings) + + while True: + parameter_input: str = InputPrompt( + message=f""" +De volgende parameters zijn nu aangemerkt voor ruimtelijke invoer: +{", ".join(parameters)} +Welke parameters wil je verwijderen? Geef deze met comma's gescheiden op. +Als er niets wordt opgegeven wordt je terug gestuurd naar het keuzemenu. +""" + ).execute() + + if parameter_input == "": + print( + BColors.OKBLUE, + "Geen parameters verwijderd.", + BColors.ENDC, + ) + return + + valid_input = all( + [ + param + for param in parameter_input.split(",") + if param in valid_list + ] + ) + + if not valid_input: + print( + BColors.OKBLUE, + f"Geen parameters toegevoegd. '{parameter_input}' bevat een ongeldige parameter.", + BColors.ENDC, + ) + continue + + break + + # Update metadata + for parameter in parameter_input.split(","): + parameters.remove(parameter) + + updated_parameters = ", ".join(parameters) + self.update_metadata_parameters(updated_parameters) + # Verwijder uit geopackage + layers: list[str] = fiona.listlayers( + self.app_settings.geopackage_filepath + ) + table_list = [ + layer + for layer in layers + if layer.split("__")[0] in updated_parameters + ] + self.remove_tables(table_list) + self.completed_update() + + def add_scenarios(self) -> None: + """ + Voeg ondergrondscenarios toe aan de lijst. Dit betekent dat alle + ruimtelijke invoer opnieuw moet worden ingeladen. + """ + # Haal de huidige scenarios op. + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_scenarios",), + ) + current_scenarios: list[str] = cursor.fetchone()[0].split(", ") + + conn.commit() + conn.close() + + # Vraag gebruiker voor toevoegingen. + while True: + scenario_input = InputPrompt( + message=f""" +De volgende scenario zijn nu toegevoegd: +{", ".join(current_scenarios)} +Welke scenarios wil je hieraan toevoegen? Geef deze met comma's gescheiden op. +Als er niets wordt opgegeven wordt je terug gestuurd naar het keuzemenu. +""" + ).execute() + + if scenario_input == "": + print( + BColors.OKBLUE, + "Geen nieuwe scenarios toegevoegd.", + BColors.ENDC, + ) + return + + # Update metadata + updated_scenarios = current_scenarios + scenario_input + self.update_metadata_scenarios(", ".join(updated_scenarios)) + + # Remove tables to recollect data. + self.remove_tables() + + def remove_scenarios(self) -> None: + """ + Verwijder ondergrondscenarios uit de lijst. Dit betekent dat alle + ruimtelijke invoer opnieuw moet worden ingeladen. + """ + # Haal de huidige scenarios op. + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_scenarios",), + ) + current_scenarios: list[str] = cursor.fetchone()[0].split(", ") + + conn.commit() + conn.close() + + # Vraag gebruiker voor toevoegingen. + while True: + scenario_input = InputPrompt( + message=f""" +De volgende scenario zijn nu toegevoegd: +{", ".join(current_scenarios)} +Welke scenarios wil je verwijderen? Geef deze met comma's gescheiden op. +Als er niets wordt opgegeven wordt je terug gestuurd naar het keuzemenu. +""" + ).execute() + + if scenario_input == "": + print( + BColors.OKBLUE, + "Geen scenarios verwijderd.", + BColors.ENDC, + ) + return + + # Update metadata + for scenario in scenario_input.split(","): + if scenario in current_scenarios: + current_scenarios.remove(scenario) + else: + print( + BColors.WARNING, + f"Scenario {scenario} niet gevonden.", + BColors.ENDC, + ) + + self.update_metadata_scenarios(", ".join(current_scenarios)) + + # Remove tables to recollect data. + self.remove_tables() + + def update_metadata_scenarios(self, scenarios: str) -> None: + """ + Update de ondergrondscenarios in de metadata. + :param scenarios: Nieuwe versie van de scenariios string voor in de metadata. + """ + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + + sql_update = "UPDATE geoprob_pipe_metadata SET metadata_value = ? WHERE metadata_type = ?" + + with conn: # transaction + conn.execute(sql_update, (scenarios, "ruimtelijke_scenarios")) + + conn.commit() + conn.close() + + def update_metadata_parameters(self, parameters: str) -> None: + """ + Update de parameters voor ruimtelijke invoer in de metadata. + + :param parameters: nieuwe versie van de parameters string voor in de metadata + """ + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + + sql_update = "UPDATE geoprob_pipe_metadata SET metadata_value = ? WHERE metadata_type = ?" + sql_insert = "INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES (?, ?)" + + with conn: # transaction + cur = conn.execute( + sql_update, (parameters, "ruimtelijke_parameters") + ) # type:ignore + if cur.rowcount == 0: + conn.execute( + sql_insert, ("ruimtelijke_parameters", parameters) + ) # type:ignore + + conn.commit() + conn.close() + + def remove_tables(self, table_list: list[str] = []) -> None: + """ + Verwijder de tabellen uit de geopackage en de metadata zodat deze opnieuw + kunnen worden ingeladen. Als de lijst leeg is worden alle tabellen met + ruimtelijke invoer verwijderd. + + :param table_list: Lijst met tabellen om te verwijderen, defaults to [] + """ + update_batch_metadata(self.app_settings, value=True) + # Voor vervangen, nieuwe scenarios of verwijderen van parameters. + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_parameters",), + ) + + parameters: list[str] = cursor.fetchone()[0].split(", ") + parameters.append("intredelijn") + layers: list[str] = fiona.listlayers( + self.app_settings.geopackage_filepath + ) + + if table_list == []: # Remove all spatial input + for layer in layers: + if layer.split("__")[0] in parameters: + cursor.execute(f"DROP TABLE IF EXISTS {layer}") + cursor.execute( + "DELETE FROM gpkg_contents WHERE table_name = ?", + (layer,), + ) + cursor.execute( + "DELETE FROM gpkg_geometry_columns WHERE table_name = ?", + (layer,), + ) + cursor.execute( + "DELETE FROM gpkg_tile_matrix_set WHERE table_name = ?", + (layer,), + ) + cursor.execute( + "DELETE FROM gpkg_tile_matrix WHERE table_name = ?", + (layer,), + ) + + else: + for layer in table_list: + if layer in layers: + cursor.execute(f"DROP TABLE IF EXISTS {layer}") + cursor.execute( + "DELETE FROM gpkg_contents WHERE table_name = ?", + (layer,), + ) + cursor.execute( + "DELETE FROM gpkg_geometry_columns WHERE table_name = ?", + (layer,), + ) + cursor.execute( + "DELETE FROM gpkg_tile_matrix_set WHERE table_name = ?", + (layer,), + ) + cursor.execute( + "DELETE FROM gpkg_tile_matrix WHERE table_name = ?", + (layer,), + ) + + conn.commit() + conn.close() + + def completed_update(self) -> None: + """ + Na aanpassing van de opties moet de applicatie opnieuw opstarten. Bied + de optie om meerdere aanpassingen te doen voordat de applicatie wordt + afgesloten. + """ + choices_list = ["Andere aanpassing uitvoeren", "Applicatie afsluiten"] + while True: + choice = ListPrompt( + message=( + "Om de aanpassingen te verwerken moet de applicatie opnieuw worden gestart. " + "Het is mogelijk om meerdere aanpassingen in een keer te doen." + ), + choices=choices_list, + default=choices_list[0], + ).execute() + + match choice: + case "Andere aanpassing uitvoeren": + self.restart_flag = True + return + + case "Applicatie afsluiten": + sys.exit("Applicatie is afgesloten.") diff --git a/geoprob_pipe/cmd_app/questionnaire.py b/geoprob_pipe/cmd_app/questionnaire.py index 34523218..573cdfa2 100644 --- a/geoprob_pipe/cmd_app/questionnaire.py +++ b/geoprob_pipe/cmd_app/questionnaire.py @@ -1,55 +1,112 @@ from __future__ import annotations -from geoprob_pipe.cmd_app.general.project import created_project -from geoprob_pipe.cmd_app.general.geohydrologisch_model import created_model -from geoprob_pipe.cmd_app.spatial_layers import ( - added_dijktraject, added_vakindeling, added_uittredepunten, added_polderpeil, added_binnenteenlijn, - added_buitenteenlijn, added_intredelijn, added_ahn, added_hrd_fragility_curves) -from geoprob_pipe.cmd_app.spatial_joins import ( - coupled_hrd_to_uittredepunten, coupled_distances_to_uittredepunten, coupled_polderpeil_to_uittredepunten, - coupled_uittredepunten_to_refline, coupled_uittredepunten_to_vakken, coupled_mv_exit_to_gis_parameter_invoer_table) -from geoprob_pipe.cmd_app.general.traject_parameters import added_traject_parameters -from geoprob_pipe.cmd_app.parameter_input.added_input_parameters import added_input_parameter_data + +import sys from typing import TYPE_CHECKING + +from geoprob_pipe.cmd_app.general.geohydrologisch_model import created_model +from geoprob_pipe.cmd_app.general.project import created_project +from geoprob_pipe.cmd_app.general.traject_parameters import ( + added_traject_parameters, +) +from geoprob_pipe.cmd_app.parameter_input.added_input_parameters import ( + added_input_parameter_data, +) from geoprob_pipe.cmd_app.run_calculations.run import run_calculations -import sys +from geoprob_pipe.cmd_app.spatial_joins import ( + coupled_distances_to_uittredepunten, + coupled_hrd_to_uittredepunten, + coupled_mv_exit_to_gis_parameter_invoer_table, + coupled_parameters_to_uittredepunten, + coupled_uittredepunten_to_refline, + coupled_uittredepunten_to_vakken, +) +from geoprob_pipe.cmd_app.spatial_layers import ( + added_ahn, + added_binnenteenlijn, + added_buitenteenlijn, + added_dijktraject, + added_hrd_fragility_curves, + added_intredelijn, + added_parameters, + added_ruimtelijke_input, + added_scenarios, + added_uittredepunten, + added_vakindeling, + check_batch_input, +) + if TYPE_CHECKING: from geoprob_pipe.cmd_app.cmd import ApplicationSettings -EARLY_EXIT_MESSAGE = f"Applicatie vroegtijdig afgesloten" +EARLY_EXIT_MESSAGE = "Applicatie vroegtijdig afgesloten" def start_questionnaire(app_settings: ApplicationSettings): - if not created_project(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) + if not created_project(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) questionnaire(app_settings=app_settings) def questionnaire(app_settings: ApplicationSettings): - print(f"\nALGEMEEN") - if not created_model(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - - print(f"\nGIS LAGEN") - if not added_dijktraject(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_vakindeling(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_hrd_fragility_curves(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_traject_parameters(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_uittredepunten(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_polderpeil(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_binnenteenlijn(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_buitenteenlijn(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not added_intredelijn(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - added_ahn(app_settings=app_settings, display_added_msg=True) # AHN may be optional - - print(f"\nGEOGRAFISCHE KOPPELINGEN") - if not coupled_uittredepunten_to_refline(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not coupled_hrd_to_uittredepunten(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not coupled_distances_to_uittredepunten(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not coupled_polderpeil_to_uittredepunten(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not coupled_uittredepunten_to_vakken(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - if not coupled_mv_exit_to_gis_parameter_invoer_table(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - - print(f"\nPARAMETER INVOER") - if not added_input_parameter_data(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) - - print(f"\nBEREKENINGEN UITVOEREN") - if not run_calculations(app_settings=app_settings): sys.exit(EARLY_EXIT_MESSAGE) + print("\nALGEMEEN") + if not created_model(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_scenarios(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_parameters(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not check_batch_input(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + + print("\nGIS LAGEN") + if not added_dijktraject(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_vakindeling(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_hrd_fragility_curves(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_traject_parameters(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_uittredepunten(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_binnenteenlijn(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_buitenteenlijn(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not added_intredelijn(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + + print("\nRUIMTELIJKE INPUT") + # Loop voor alle parameters + if not added_ruimtelijke_input(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + added_ahn( + app_settings=app_settings, display_added_msg=True + ) # AHN may be optional + + print("\nGEOGRAFISCHE KOPPELINGEN") + if not coupled_uittredepunten_to_refline(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not coupled_distances_to_uittredepunten(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not coupled_uittredepunten_to_vakken(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not coupled_hrd_to_uittredepunten(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + if not coupled_mv_exit_to_gis_parameter_invoer_table( + app_settings=app_settings + ): + sys.exit(EARLY_EXIT_MESSAGE) + # TODO Later Should: make coupling for raster if import is added. + if not coupled_parameters_to_uittredepunten(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + + print("\nPARAMETER INVOER") + # TODO VJ: Vraag CP waar de traject defaults automatisch verwijderd worden + if not added_input_parameter_data(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) + + print("\nBEREKENINGEN UITVOEREN") + if not run_calculations(app_settings=app_settings): + sys.exit(EARLY_EXIT_MESSAGE) diff --git a/geoprob_pipe/cmd_app/run_calculations/run.py b/geoprob_pipe/cmd_app/run_calculations/run.py index 7c9ae18c..0663988d 100644 --- a/geoprob_pipe/cmd_app/run_calculations/run.py +++ b/geoprob_pipe/cmd_app/run_calculations/run.py @@ -40,9 +40,7 @@ def _convert_vak_ids_str_to_ints(input_str: str) -> Tuple[List[int], List[str]]: return list(set(list_vak_ids_int)), unknown_inputs def request_vakken_to_run() -> str: - list_vak_ids_int: List[int] = [] - vakken_input_is_valid = False - while vakken_input_is_valid is False: + while True: vakken_input: str = inquirer.text( message="Specificeer welke vakken je wilt doorrekenen. Doe dit door comma-separated de vak nummers (id) " "op te geven. Bijvoorbeeld '4,5,6,7,10-15'.", @@ -62,7 +60,7 @@ def request_vakken_to_run() -> str: f"Probeer opnieuw.{BColors.ENDC}") continue - vakken_input_is_valid = True + break return f"vakken:{','.join([str(item) for item in list_vak_ids_int])}" @@ -82,7 +80,7 @@ def run_calculations(app_settings: ApplicationSettings) -> bool: if choice == choices_list[0]: geoprob_pipe = GeoProbPipe(app_settings) geoprob_pipe.export_archive() - print(BColors.OKBLUE, f"✅ Berekeningen zijn uitgevoerd.", BColors.ENDC) + print(BColors.OKBLUE, "✅ Berekeningen zijn uitgevoerd.", BColors.ENDC) elif choice == choices_list[1]: vakken_str: str = request_vakken_to_run() app_settings.to_run = vakken_str @@ -91,7 +89,7 @@ def run_calculations(app_settings: ApplicationSettings) -> bool: print(BColors.OKBLUE, f"✅ Berekeningen zijn uitgevoerd, voor vakken " f"{app_settings.to_run.replace("vakken:", "")}.", BColors.ENDC) elif choice == choices_list[2]: - sys.exit(f"Applicatie is afgesloten.") + sys.exit("Applicatie is afgesloten.") else: raise ValueError return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/__init__.py b/geoprob_pipe/cmd_app/spatial_joins/__init__.py index 73a4725f..0527ff01 100644 --- a/geoprob_pipe/cmd_app/spatial_joins/__init__.py +++ b/geoprob_pipe/cmd_app/spatial_joins/__init__.py @@ -1,7 +1,8 @@ -from geoprob_pipe.cmd_app.spatial_joins.coupled_hrd import coupled_hrd_to_uittredepunten -from geoprob_pipe.cmd_app.spatial_joins.coupled_distance_to_exit_points import ( - coupled_distances_to_uittredepunten) -from geoprob_pipe.cmd_app.spatial_joins.coupled_polderpeil import coupled_polderpeil_to_uittredepunten -from geoprob_pipe.cmd_app.spatial_joins.coupled_exit_points_to_refline import coupled_uittredepunten_to_refline -from geoprob_pipe.cmd_app.spatial_joins.coupled_exit_points_to_vakken import coupled_uittredepunten_to_vakken -from geoprob_pipe.cmd_app.spatial_joins.coupled_mv_exit import coupled_mv_exit_to_gis_parameter_invoer_table +from .distance import ( + coupled_distances_to_uittredepunten, +) +from .refline import coupled_uittredepunten_to_refline +from .vakken import coupled_uittredepunten_to_vakken +from .hrd import coupled_hrd_to_uittredepunten +from .mv_exit import coupled_mv_exit_to_gis_parameter_invoer_table +from .parameters import coupled_parameters_to_uittredepunten diff --git a/geoprob_pipe/cmd_app/spatial_joins/couple_objects/__init__.py b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/geoprob_pipe/cmd_app/spatial_joins/couple_objects/base.py b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/base.py new file mode 100644 index 00000000..b43faf33 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/base.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import datetime +import sqlite3 +from typing import TYPE_CHECKING + +import numpy as np +import pandas as pd + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +class BaseCouple: + """ + Base class voor het koppelen van GIS-kagen aan de uittredepunten en deze op + de correcte manier toe te voegen aan de geopackage. + """ + + # Placeholders + app_settings: ApplicationSettings + param: str + + def _create_df(self, join_df: pd.DataFrame, scenario: str) -> pd.DataFrame: + """ + Maak de dataframe die moet worden toegevoegd aan de geopackage. + Als er geen waarde is wordt een default gebruikt. + + :param join_df: DataFrame met de data uit de join. + :param scenario: Ondergrondscenario_naam voor in de dataframe. Kan "" zijn. + :return: Dataframe met juiste kolommen. + """ + df = join_df[["uittredepunt_id"]].copy() + df = df.rename(columns={"uittredepunt_id": "scope_referentie"}) + df["parameter"] = self.param + df["scope"] = "uittredepunt" + df["ondergrondscenario_naam"] = scenario + df["distribution_type"] = join_df.get(f"{self.param}__dist", "deterministic") + df["mean"] = join_df.get(f"{self.param}__mean", np.nan) + df["variation"] = join_df.get(f"{self.param}__var", np.nan) + df["deviation"] = join_df.get(f"{self.param}__dev", np.nan) + df["minimum"] = join_df.get(f"{self.param}__min", np.nan) + df["maximum"] = join_df.get(f"{self.param}__max", np.nan) + df["fragility_values_ref"] = "" + df["bronnen"] = "" + df["opmerking"] = "" + + # Sort columns + df = df[ + [ + "parameter", + "scope", + "scope_referentie", + "ondergrondscenario_naam", + "distribution_type", + "mean", + "variation", + "deviation", + "minimum", + "maximum", + "fragility_values_ref", + "bronnen", + "opmerking", + ] + ] + return df + + def _upsert_to_gpkg(self, df: pd.DataFrame): + """ + Voeg de dataframe toe aan de tabel. Als de tabel nog niet bestaat + wordt deze op de correcte manier opgezet. Met een unique verzameling + waarden of te kunnen overschrijven. Om te zorgen dat de tabel geschikt + is voor een upsert moet er een set van kolommen uniek zijn. Dit is + lastiger om automatisch te doen met `.to_sql`. Hier wordt de tabel + ook handmatig toegevoegd aan gpkg_contents. + + :param df: DataFrame met juiste kolommen. + """ + conn = sqlite3.connect(self.app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS data__gis_parameter_invoer ( + fid INTEGER PRIMARY KEY AUTOINCREMENT, + parameter TEXT, + scope TEXT, + scope_referentie INTEGER, + ondergrondscenario_naam TEXT, + distribution_type TEXT, + mean REAL, + variation REAL, + deviation REAL, + minimum REAL, + maximum REAL, + fragility_values_ref TEXT, + bronnen TEXT, + opmerking TEXT, + UNIQUE(parameter, scope, scope_referentie, ondergrondscenario_naam) + ) + """ + ) + data = [tuple(row) for row in df.to_numpy()] + placeholders = ", ".join(["?"] * 13) + # UPSERT naar tabel + cursor.executemany( + f""" + INSERT INTO data__gis_parameter_invoer ( + parameter, + scope, + scope_referentie, + ondergrondscenario_naam, + distribution_type, + mean, + variation, + deviation, + minimum, + maximum, + fragility_values_ref, + bronnen, + opmerking + ) + VALUES ({placeholders}) + ON CONFLICT ( + parameter, scope, scope_referentie, ondergrondscenario_naam + ) DO UPDATE SET + distribution_type = excluded.distribution_type, + mean = excluded.mean, + variation = excluded.variation, + deviation = excluded.deviation, + minimum = excluded.minimum, + maximum = excluded.maximum, + fragility_values_ref = excluded.fragility_values_ref, + bronnen = excluded.bronnen, + opmerking = excluded.opmerking + """, + data, + ) + # UPSERT naar gpkg_contents, vereist in gpkg om leesbaar te zijn + # voor GIS software + content = ( + "data__gis_parameter_invoer", + "attributes", + "data__gis_parameter_invoer", + "", + datetime.datetime.now(), + 0, + ) + cursor.execute( + """ + INSERT INTO gpkg_contents ( + table_name, + data_type, + identifier, + description, + last_change, + srs_id + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (identifier) DO UPDATE SET + table_name = excluded.table_name, + data_type = excluded.data_type, + description = excluded.description, + last_change = excluded.last_change, + srs_id = excluded.srs_id + """, + content + ) + + conn.commit() + conn.close() \ No newline at end of file diff --git a/geoprob_pipe/cmd_app/spatial_joins/couple_objects/distance.py b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/distance.py new file mode 100644 index 00000000..dfa96ac6 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/distance.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import pandas as pd +from typing import TYPE_CHECKING + +from .base import BaseCouple + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +class DistCouple(BaseCouple): + def __init__( + self, app_settings: ApplicationSettings, param: str, df: pd.DataFrame + ) -> None: + """ + Class om de afstand tussen de uittredepunten en de lijnen toe te voegen + aan de geopackage. + + :param app_settings: Object met alle settings van de applicatie. + :param str param: De parameter die wordt toegevoegd aan de tabel. + :param pd.DataFrame df: Dataframe met de waardes per uittredepunt + """ + self.app_settings = app_settings + self.param = param + self.df = df + + def couple_exit_points(self, scenario: str = ""): + """ + Methode voer het correct opstellen van de dataframe en deze aan de + geopackage toe te voegen. + + :param str scenario: Naam van het ondergrondscenario waar deze waardes voor + gelden, defaults to "" + """ + df_to_add = self._create_df(self.df, scenario=scenario) + self._upsert_to_gpkg(df_to_add) diff --git a/geoprob_pipe/cmd_app/spatial_joins/couple_objects/hrd.py b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/hrd.py new file mode 100644 index 00000000..f0c4924c --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/hrd.py @@ -0,0 +1,85 @@ +""" +Class voor het koppelen van de HRD-locaties aan de uittredepunten. +Versimpelde versie van de ShapeCouple class. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import geopandas as gpd +import pandas as pd +import numpy as np + +from .base import BaseCouple + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +class HRDCouple(BaseCouple): + def __init__(self, app_settings: ApplicationSettings) -> None: + """ + Class voor het koppelen van de hrd locatie namen aan de uittredepunten. + + :param app_settings: Object met de settings van de applicatie. + """ + self.app_settings = app_settings + + def couple_exit_points(self) -> None: + """ + Koppel de hrd_locaties aan de uittredepunten. En voeg deze toe aan de + geopackage op de juiste manier. + """ + # Read uittredepunten + gdf_exit_points: gpd.GeoDataFrame = gpd.read_file( + self.app_settings.geopackage_filepath, layer="geom__uittredepunten" + ) + gdf_hrd: gpd.GeoDataFrame = gpd.read_file( + self.app_settings.geopackage_filepath, layer="geom__hrd_locaties" + ) + join_df = gdf_exit_points.sjoin_nearest(gdf_hrd, how="left") + df_to_add = self._create_df_hrd(join_df) + self._upsert_to_gpkg(df_to_add) + + def _create_df_hrd(self, join_df: pd.DataFrame) -> pd.DataFrame: + """ + Methode om de dataframe op te zetten voor de hrd locatie invoer. + + :param join_df: _description_ + :return: _description_ + """ + df = join_df[["uittredepunt_id"]].copy() + df = df.rename(columns={"uittredepunt_id": "scope_referentie"}) + df["parameter"] = "buitenwaterstand" + df["scope"] = "uittredepunt" + df["ondergrondscenario_naam"] = "" + df["distribution_type"] = "cdf_curve" + df["mean"] = np.nan + df["variation"] = np.nan + df["deviation"] = np.nan + df["minimum"] = np.nan + df["maximum"] = np.nan + df["fragility_values_ref"] = join_df.get("location_name", "") + df["bronnen"] = "" + df["opmerking"] = "" + + # Sort columns + df = df[ + [ + "parameter", + "scope", + "scope_referentie", + "ondergrondscenario_naam", + "distribution_type", + "mean", + "variation", + "deviation", + "minimum", + "maximum", + "fragility_values_ref", + "bronnen", + "opmerking", + ] + ] + return df diff --git a/geoprob_pipe/cmd_app/spatial_joins/couple_objects/shape.py b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/shape.py new file mode 100644 index 00000000..37cc4bed --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_joins/couple_objects/shape.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +import geopandas as gpd + +from geoprob_pipe.cmd_app.spatial_layers import LIST_PARAMS +from geoprob_pipe.utils.validation_messages import BColors + +from .base import BaseCouple + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +class ShapeCouple(BaseCouple): + def __init__(self, app_settings: ApplicationSettings, param: str) -> None: + """ + Class om de uittredepunten te koppelen aan een gis-laag vanuit de geopackage. + De gekoppelde waarden worden weggeschreven in de tabel 'gis_join_parameter_invoer'. + + :param app_settings: `ApplicationSettings` object. + :param param: Parameter die gekoppeld moet worden. + """ + self.app_settings = app_settings + self.param = param + conn = sqlite3.connect(app_settings.geopackage_filepath) + cursor = conn.cursor() + # Get scenarios to add + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_scenarios",), + ) + self.scenarios: list[str] = cursor.fetchone()[0].split(", ") + + # Get table names + cursor.execute( + "SELECT name FROM sqlite_master WHERE type=?;", ("table",) + ) + self.tables_names: list[str] = [row[0] for row in cursor.fetchall()] + conn.close() + + def couple_exit_points(self): + """ + Lees de uittredepunten uit de geopackage en bepaal of er meerdere scenarios + zijn om toe te voegen. + + :raises KeyError: Als de parameter tabel niet wordt gevonden. + """ + # Read uittredepunten + self.gdf_exit_points: gpd.GeoDataFrame = gpd.read_file( + self.app_settings.geopackage_filepath, layer="geom__uittredepunten" + ) + + self.param_tables = [ + table + for table in self.tables_names + if table.strip("geom_").split("__")[0] == self.param + ] + + if len(self.param_tables) == 0: + raise KeyError( + f"{self.param} niet gevonden in de tabellen van de geopackage." + ) + + elif len(self.param_tables) == 1: + # Geen invoer per scenario + self._add_single_layer() + + else: + # Invoer per scenario + self._add_multiple_layers() + + def _add_single_layer(self): + """ + Voeg een laag toe via koppeling, met de uittredepunten als er geen + ruimtelijke scenarios zijn voor de parameter. + """ + gdf_parameter: gpd.GeoDataFrame = gpd.read_file( + self.app_settings.geopackage_filepath, layer=f"geom__{self.param}" + ) + self._check_shape(gdf_parameter) + + def _add_multiple_layers(self): + """ + Voeg de lagen een voor een toe als er ruimtelijke scenarios zijn voor + de parameter. + """ + for layer in self.param_tables: + gdf_parameter: gpd.GeoDataFrame = gpd.read_file( + self.app_settings.geopackage_filepath, + layer=f"geom__{layer}", + ) + scenario = layer.split("__")[1] + self._check_shape(gdf_parameter, scenario) + + def _check_shape(self, gdf: gpd.GeoDataFrame, scenario: str = ""): + """ + Bepaal welke shapes er in de laag zitten en of deze een geaccepteerde + optie zijn. + + :param gdf: GeoDataFrame met de ruimtelijke invoer. + :param scenario: Ondergrondscenario_naam als invoer per scenario voor + deze parameters is gekozen, defaults to "" + :raises KeyError: Lijn niet geaccepteerd bij deze parameter. + :raises KeyError: Polygon niet geaccepteerd bij deze parameter. + :raises ImportError: De laag bestaat uit verschillende geometrie types. + """ + if (gdf.geom_type.isin(["LineString", "MultiLineString"])).all(): + if "line" in LIST_PARAMS[self.param]["shape"]: + self._join_to_line(gdf, scenario) + else: + raise KeyError( + "Deze parameter is niet geschikt om aan een lijn gekoppeld te worden." + ) + elif (gdf.geom_type.isin(["Polygon", "MultiPolygon"])).all(): + if "polygon" in LIST_PARAMS[self.param]["shape"]: + self._join_to_polygon(gdf, scenario) + else: + raise KeyError( + "Deze parameter is niet geschikt om aan een polygon gekoppeld te worden." + ) + else: + raise ImportError( + "De geïmporteerde laag bestaat niet uit alleen lijnen of uit alleen polygonen." + ) + + def _join_to_line(self, gdf: gpd.GeoDataFrame, scenario: str): + """ + Maak een spatial join van de uittredepunten met een lijn. + + :param gdf: GeoDataFrame met de ruimtelijke invoer. + :param scenario: Ondergrondscenario_naam voor in de dataframe. Kan "" zijn. + """ + join_gdf = self.gdf_exit_points.sjoin_nearest(gdf, how="left").drop_duplicates("uittredepunt_id") + df_to_add = self._create_df(join_gdf, scenario) + + self._upsert_to_gpkg(df_to_add) + + def _join_to_polygon(self, gdf: gpd.GeoDataFrame, scenario: str): + """ + Maak een spatial join van de uittredepunten met een polygon. + + :param gdf: GeoDataFrame met de ruimtelijke invoer. + :param scenario: Ondergrondscenario_naam voor in de dataframe. Kan "" zijn. + """ + join_gdf = self.gdf_exit_points.sjoin(gdf, how="left") + outside = join_gdf[join_gdf.index_right.isna()] + if len(outside) != 0: + print( + BColors.WARNING, + f"{len(outside)} uittredepunten vallen buiten de polygonen bij {self.param}.", + BColors.ENDC + ) + join_gdf = join_gdf[join_gdf.index_right.notna()] + df_to_add = self._create_df(join_gdf, scenario) + + self._upsert_to_gpkg(df_to_add) + + # TODO Later Should: Add option for raster import and coupling. + def _join_to_raster(self): + raise NotImplementedError("Toevoegen van parameters via raster is nog niet mogelijk.") diff --git a/geoprob_pipe/cmd_app/spatial_joins/coupled_distance_to_exit_points.py b/geoprob_pipe/cmd_app/spatial_joins/coupled_distance_to_exit_points.py deleted file mode 100644 index 7bcd785b..00000000 --- a/geoprob_pipe/cmd_app/spatial_joins/coupled_distance_to_exit_points.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -TODO Nu Must Klein: Voeg laag toe aan GeoPackage met visuele koppeling tussen intrede en uittredepunten. -""" - -from __future__ import annotations -from geopandas import GeoDataFrame, read_file -from pathlib import Path -from geoprob_pipe.cmd_app.spatial_joins.utils import append_to_gis_join_parameter_invoer_table -from geoprob_pipe.utils.validation_messages import BColors -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from geoprob_pipe.cmd_app.cmd import ApplicationSettings - - -def coupled_distances_to_uittredepunten(app_settings: ApplicationSettings) -> bool: - - # Read uittredepunten - gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") - - # Check if already added - columns = gdf_exit_points.columns - if "afstand_intredelijn" in columns and "afstand_buitenteenlijn" in columns and "afstand_binnenteenlijn" in columns: - print(BColors.OKBLUE, - f"✔ Afstanden intrede, buitenteen en binnenteen al gekoppeld aan uittredepunten.", BColors.ENDC) - return True # Assuming already added - # TODO: Uitfaseren dat deze kolommen gekoppeld worden aan de exit_points tabel - - # Distance to intredelijn - gdf_intredelijnen: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="intredelijn") - gdf_exit_points['afstand_intredelijn'] = gdf_exit_points.geometry.apply( - lambda pnt: round(gdf_intredelijnen.distance(pnt).min(), 1)) - df_l_intrede = gdf_exit_points[["uittredepunt_id", "afstand_intredelijn"]] - df_l_intrede = df_l_intrede.rename(columns={"afstand_intredelijn": "mean"}) - append_to_gis_join_parameter_invoer_table( - df_sjoin=df_l_intrede, parameter_name="L_intrede", geopackage_filepath=app_settings.geopackage_filepath) - - # Distance to buitenteenlijn - gdf_buitenteenlijnen: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="buitenteenlijn") - gdf_exit_points['afstand_buitenteenlijn'] = gdf_exit_points.geometry.apply( - lambda pnt: round(gdf_buitenteenlijnen.distance(pnt).min(), 1)) - df_l_but = gdf_exit_points[["uittredepunt_id", "afstand_buitenteenlijn"]] - df_l_but = df_l_but.rename(columns={"afstand_buitenteenlijn": "mean"}) - append_to_gis_join_parameter_invoer_table( - df_sjoin=df_l_but, parameter_name="L_but", geopackage_filepath=app_settings.geopackage_filepath) - - # Distance to binnenteenlijn - gdf_binnenteenlijn: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="binnenteenlijn") - gdf_exit_points['afstand_binnenteenlijn'] = gdf_exit_points.geometry.apply( - lambda pnt: round(gdf_binnenteenlijn.distance(pnt).min(), 1)) - df_l_intrede = gdf_exit_points[["uittredepunt_id", "afstand_binnenteenlijn"]] - df_l_intrede = df_l_intrede.rename(columns={"afstand_binnenteenlijn": "mean"}) - append_to_gis_join_parameter_invoer_table( - df_sjoin=df_l_intrede, parameter_name="L_bit", geopackage_filepath=app_settings.geopackage_filepath) - - # Store back in geopackage - gdf_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="uittredepunten", driver="GPKG") - print(BColors.OKBLUE, - f"✅ Afstanden intrede, buitenteen en binnenteen zijn nu gekoppeld aan de uittredepunten.", BColors.ENDC) - return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py b/geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py deleted file mode 100644 index 371f2d16..00000000 --- a/geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -TODO Nu Must Klein: Voeg laag toe aan GeoPackage met visuele koppeling tussen HRD en uittredepunten. -""" - -from __future__ import annotations -from geopandas import GeoDataFrame, read_file -import fiona -from pathlib import Path -from geoprob_pipe.cmd_app.spatial_joins.utils import append_hrd_to_gis_join_parameter_invoer_table -from geoprob_pipe.utils.validation_messages import BColors -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from geoprob_pipe.cmd_app.cmd import ApplicationSettings - - -def coupled_hrd_to_uittredepunten(app_settings: ApplicationSettings) -> bool: - - layers = fiona.listlayers(app_settings.geopackage_filepath) - - if "hrd_locaties" not in layers: - return True # Geen Hydra-locatie geïmporteerd, dan ook niks koppelen. - - # Getting necessary GeoDataframes - gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") - gdf_hrd_locations: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="hrd_locaties") - - # Check if already added - if "hrd_name" in gdf_exit_points.columns: - print(BColors.OKBLUE, f"✔ HRD-locaties al gekoppeld aan uittredepunten.", BColors.ENDC) - return True # Assuming already added - - # Perform spatial join to find the nearest HRD-location for each Exit Point - gdf_exit_with_hrd = gdf_exit_points.sjoin_nearest( - gdf_hrd_locations[['geometry', 'location_name']], how='left', distance_col='distance') - - # Define which columns to keep (after spatial join) - columns_to_keep = list(gdf_exit_points.columns) - columns_to_keep.append("location_name") - gdf_new_exit_points = gdf_exit_with_hrd[columns_to_keep] - gdf_new_exit_points = gdf_new_exit_points.rename(columns={"location_name": "hrd_name"}) - append_hrd_to_gis_join_parameter_invoer_table( - df_sjoin=gdf_new_exit_points[["uittredepunt_id", "hrd_name"]], - geopackage_filepath=app_settings.geopackage_filepath) - - # Store back in geopackage - gdf_new_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="uittredepunten", driver="GPKG") - # TODO: Uitfaseren dat HRD-koppeling middels direct aan de geometrie van het uittredepunt zit. - print(BColors.OKBLUE, f"✅ HRD-locaties zijn nu gekoppeld aan de uittredepunten.", BColors.ENDC) - return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/coupled_polderpeil.py b/geoprob_pipe/cmd_app/spatial_joins/coupled_polderpeil.py index acc23c5d..0899cde0 100644 --- a/geoprob_pipe/cmd_app/spatial_joins/coupled_polderpeil.py +++ b/geoprob_pipe/cmd_app/spatial_joins/coupled_polderpeil.py @@ -1,49 +1,49 @@ """ TODO Nu Must Klein: Voeg laag toe aan GeoPackage met visuele koppeling tussen HRD en uittredepunten. """ - -from __future__ import annotations -from geopandas import GeoDataFrame, read_file -from pathlib import Path -from pandas import DataFrame -from geoprob_pipe.cmd_app.spatial_joins.utils import append_to_gis_join_parameter_invoer_table -from geoprob_pipe.utils.validation_messages import BColors -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from geoprob_pipe.cmd_app.cmd import ApplicationSettings - - -def coupled_polderpeil_to_uittredepunten(app_settings: ApplicationSettings) -> bool: - - - # Getting necessary GeoDataframes - gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") - gdf_polderpeil: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="polderpeil") - - # Check if already added - if "polderpeil" in gdf_exit_points.columns: - print(BColors.OKBLUE, f"✔ Polderpeil al gekoppeld aan uittredepunten.", BColors.ENDC) - return True # Assuming already added - - # Perform spatial join to find the nearest HRD-location for each Exit Point - gdf_exit_with_hrd = gdf_exit_points.sjoin_nearest( - gdf_polderpeil[['geometry', 'polderpeil']], how='left', distance_col='distance') - - # Define which columns to keep (after spatial join) - columns_to_keep = list(gdf_exit_points.columns) - columns_to_keep.append("polderpeil") - gdf_new_exit_points = gdf_exit_with_hrd[columns_to_keep] - - # Append to new version of geospatial storage - df_to_append: DataFrame = gdf_new_exit_points[["polderpeil", "uittredepunt_id"]] - df_to_append = df_to_append.rename(columns={"polderpeil": "mean"}) - append_to_gis_join_parameter_invoer_table( - df_sjoin=df_to_append, - parameter_name="polderpeil", - geopackage_filepath=app_settings.geopackage_filepath) - - # Store back in geopackage - gdf_new_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="uittredepunten", driver="GPKG") - # TODO: Uitfaseren deze oude manier van polderpeil opslaan - print(BColors.OKBLUE, f"✅ Polderpeil is nu gekoppeld aan de uittredepunten.", BColors.ENDC) - return True +# Vervallen +# from __future__ import annotations +# from geopandas import GeoDataFrame, read_file +# from pathlib import Path +# from pandas import DataFrame +# from geoprob_pipe.cmd_app.spatial_joins.utils import append_to_gis_join_parameter_invoer_table +# from geoprob_pipe.utils.validation_messages import BColors +# from typing import TYPE_CHECKING +# if TYPE_CHECKING: +# from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +# def coupled_polderpeil_to_uittredepunten(app_settings: ApplicationSettings) -> bool: + + +# # Getting necessary GeoDataframes +# gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") +# gdf_polderpeil: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="polderpeil") + +# # Check if already added +# if "polderpeil" in gdf_exit_points.columns: +# print(BColors.OKBLUE, f"✔ Polderpeil al gekoppeld aan uittredepunten.", BColors.ENDC) +# return True # Assuming already added + +# # Perform spatial join to find the nearest HRD-location for each Exit Point +# gdf_exit_with_hrd = gdf_exit_points.sjoin_nearest( +# gdf_polderpeil[['geometry', 'polderpeil']], how='left', distance_col='distance') + +# # Define which columns to keep (after spatial join) +# columns_to_keep = list(gdf_exit_points.columns) +# columns_to_keep.append("polderpeil") +# gdf_new_exit_points = gdf_exit_with_hrd[columns_to_keep] + +# # Append to new version of geospatial storage +# df_to_append: DataFrame = gdf_new_exit_points[["polderpeil", "uittredepunt_id"]] +# df_to_append = df_to_append.rename(columns={"polderpeil": "mean"}) +# append_to_gis_join_parameter_invoer_table( +# df_sjoin=df_to_append, +# parameter_name="polderpeil", +# geopackage_filepath=app_settings.geopackage_filepath) + +# # Store back in geopackage +# gdf_new_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="uittredepunten", driver="GPKG") +# # TODO: Uitfaseren deze oude manier van polderpeil opslaan +# print(BColors.OKBLUE, f"✅ Polderpeil is nu gekoppeld aan de uittredepunten.", BColors.ENDC) +# return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/distance.py b/geoprob_pipe/cmd_app/spatial_joins/distance.py new file mode 100644 index 00000000..13cb072f --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_joins/distance.py @@ -0,0 +1,116 @@ +""" +TODO Nu Must Klein: Voeg laag toe aan GeoPackage met visuele koppeling tussen intrede en uittredepunten. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import geopandas as gpd +import fiona + +from geoprob_pipe.cmd_app.spatial_joins.couple_objects.distance import ( + DistCouple, +) +from geoprob_pipe.utils.validation_messages import BColors + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def coupled_distances_to_uittredepunten( + app_settings: ApplicationSettings, +) -> bool: + """ + Bepaal de afstanden tussen de uittredepunten en de intrede, binnenteen en + buitenteenlijnen. Als deze er al in staan worden ze overschreven. + + :param app_settings: Object met de instellingen van de applicatie + """ + + # Lees uittredepunten uit gpkg + gdf_exit_points: gpd.GeoDataFrame = gpd.read_file( + app_settings.geopackage_filepath, layer="geom__uittredepunten" + ) + + # Distance to intredelijn + layers: list[str] = fiona.listlayers(app_settings.geopackage_filepath) + # Lees de intrede lijn(en) uit. Als er meerdere zijn worden deze allemaal + # uitgelezen. Er kan zowel een intredelijn zonder scenario als met een scenario + # tegelijk zijn. + for layer in layers: + if "geom__intredelijn" in layers: # Een intredelijn voor alle scenarios + gdf_intredelijnen: gpd.GeoDataFrame = gpd.read_file( + app_settings.geopackage_filepath, layer="geom__intredelijn" + ) + gdf_exit_points["afstand_intredelijn"] = ( + gdf_exit_points.geometry.apply( + lambda pnt: round(gdf_intredelijnen.distance(pnt).min(), 1) + ) + ) + df_l_intrede = gdf_exit_points[ + ["uittredepunt_id", "afstand_intredelijn"] + ] + df_l_intrede = df_l_intrede.rename( + columns={"afstand_intredelijn": "L_intrede__mean"} + ) + DistCouple( + app_settings, "L_intrede", df_l_intrede + ).couple_exit_points() + + # Intrede lijn per scenario + if len(layer.split("__")) == 3 and "intredelijn" in layer.split("__"): + gdf_intredelijnen: gpd.GeoDataFrame = gpd.read_file( + app_settings.geopackage_filepath, layer=layer + ) + gdf_exit_points["afstand_intredelijn"] = ( + gdf_exit_points.geometry.apply( + lambda pnt: round( + gdf_intredelijnen.distance(pnt).min(), 1 + ) + ) + ) + df_l_intrede = gdf_exit_points[ + ["uittredepunt_id", "afstand_intredelijn"] + ] + df_l_intrede = df_l_intrede.rename( + columns={"afstand_intredelijn": "L_intrede__mean"} + ) + DistCouple( + app_settings, "L_intrede", df_l_intrede + ).couple_exit_points(scenario=layer.split("__")[2]) + + # Distance to buitenteenlijn + gdf_buitenteenlijnen: gpd.GeoDataFrame = gpd.read_file( + app_settings.geopackage_filepath, layer="geom__buitenteenlijn" + ) + gdf_exit_points["afstand_buitenteenlijn"] = gdf_exit_points.geometry.apply( + lambda pnt: round(gdf_buitenteenlijnen.distance(pnt).min(), 1) + ) + df_l_but = gdf_exit_points[["uittredepunt_id", "afstand_buitenteenlijn"]] + df_l_but = df_l_but.rename( + columns={"afstand_buitenteenlijn": "L_but__mean"} + ) + + DistCouple(app_settings, "L_but", df_l_but).couple_exit_points() + + # Distance to binnenteenlijn + gdf_binnenteenlijn: gpd.GeoDataFrame = gpd.read_file( + app_settings.geopackage_filepath, layer="geom__binnenteenlijn" + ) + gdf_exit_points["afstand_binnenteenlijn"] = gdf_exit_points.geometry.apply( + lambda pnt: round(gdf_binnenteenlijn.distance(pnt).min(), 1) + ) + df_l_bit = gdf_exit_points[["uittredepunt_id", "afstand_binnenteenlijn"]] + df_l_bit = df_l_bit.rename( + columns={"afstand_binnenteenlijn": "L_bit__mean"} + ) + + DistCouple(app_settings, "L_bit", df_l_bit).couple_exit_points() + + print( + BColors.OKBLUE, + "✔ Afstanden intrede, buitenteen en binnenteen zijn nu gekoppeld aan de uittredepunten.", + BColors.ENDC, + ) + return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/hrd.py b/geoprob_pipe/cmd_app/spatial_joins/hrd.py new file mode 100644 index 00000000..7fd0a8c4 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_joins/hrd.py @@ -0,0 +1,33 @@ +""" +TODO Nu Must Klein: Voeg laag toe aan GeoPackage met visuele koppeling tussen HRD en uittredepunten. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import fiona +from .couple_objects.hrd import HRDCouple + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def coupled_hrd_to_uittredepunten(app_settings: ApplicationSettings) -> bool: + """ + Voeg de HRD locaties toe aan de invoer tabel als deze zijn opgegeven. + Als deze er al staan worden ze overschreven. + + :param ApplicationSettings app_settings: Object met de instellingen van de applicatie + :return: _description_ + :rtype: bool + """ + + layers = fiona.listlayers(app_settings.geopackage_filepath) + + if "geom__hrd_locaties" not in layers: + return True # Geen Hydra-locatie geïmporteerd, dan ook niks koppelen. + + HRDCouple(app_settings).couple_exit_points() + + return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/coupled_mv_exit.py b/geoprob_pipe/cmd_app/spatial_joins/mv_exit.py similarity index 85% rename from geoprob_pipe/cmd_app/spatial_joins/coupled_mv_exit.py rename to geoprob_pipe/cmd_app/spatial_joins/mv_exit.py index 50b467cf..14f827de 100644 --- a/geoprob_pipe/cmd_app/spatial_joins/coupled_mv_exit.py +++ b/geoprob_pipe/cmd_app/spatial_joins/mv_exit.py @@ -12,14 +12,14 @@ def coupled_mv_exit_to_gis_parameter_invoer_table(app_settings: ApplicationSetti # Check if already added, if so skip conn = sqlite3.connect(app_settings.geopackage_filepath) - df_existing = read_sql_query("SELECT * FROM gis_join_parameter_invoer;", conn) + df_existing = read_sql_query("SELECT * FROM data__gis_parameter_invoer;", conn) if "mv_exit" in df_existing['parameter'].unique(): return True # Getting necessary GeoDataframes - gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") + gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__uittredepunten") - # Append to new version of geospatial storage + # Append to new version of geo-spatial storage df_to_append: DataFrame = gdf_exit_points[["mv_exit", "uittredepunt_id"]] df_to_append = df_to_append.rename(columns={"mv_exit": "mean"}) append_to_gis_join_parameter_invoer_table( diff --git a/geoprob_pipe/cmd_app/spatial_joins/parameters.py b/geoprob_pipe/cmd_app/spatial_joins/parameters.py new file mode 100644 index 00000000..9f0c8059 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_joins/parameters.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +from .couple_objects.shape import ShapeCouple +from geoprob_pipe.utils.validation_messages import BColors + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def coupled_parameters_to_uittredepunten( + app_settings: ApplicationSettings, +) -> bool: + conn = sqlite3.connect(app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_parameters",), + ) + parameters: list[str] = cursor.fetchone()[0].split(", ") + conn.close() + if parameters == ['']: # De split maakt een lege string, als de uit de metadata gelezen string leeg is. + print( + BColors.OKBLUE, + "✔ Geen ruimtelijke koppelingen.", + BColors.ENDC, + ) + return True + + for parameter in parameters: + ShapeCouple(app_settings, parameter).couple_exit_points() + + return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/coupled_exit_points_to_refline.py b/geoprob_pipe/cmd_app/spatial_joins/refline.py similarity index 94% rename from geoprob_pipe/cmd_app/spatial_joins/coupled_exit_points_to_refline.py rename to geoprob_pipe/cmd_app/spatial_joins/refline.py index 05dee1eb..dec75581 100644 --- a/geoprob_pipe/cmd_app/spatial_joins/coupled_exit_points_to_refline.py +++ b/geoprob_pipe/cmd_app/spatial_joins/refline.py @@ -13,14 +13,14 @@ def coupled_uittredepunten_to_refline(app_settings: ApplicationSettings) -> bool functie dat automatisch. """ # Read uittredepunten - gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") + gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__uittredepunten") # Check if already added if "metrering" in gdf_exit_points.columns and "afstand_reflijn" in gdf_exit_points.columns: print(BColors.OKBLUE, f"✔ Afstand en metrering tot reflijn al gekoppeld aan uittredepunten.", BColors.ENDC) return True # Assuming already added - gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="dijktraject") + gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__dijktraject") gdf_dijktraject_geom = gdf_dijktraject.iloc[0].geometry if isinstance(gdf_dijktraject_geom, MultiLineString): assert gdf_dijktraject_geom.geoms.__len__() == 1 @@ -41,7 +41,7 @@ def coupled_uittredepunten_to_refline(app_settings: ApplicationSettings) -> bool gdf_exit_points['uittredepunt_id'] = range(1, len(gdf_exit_points) + 1) # Store back in geopackage - gdf_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="uittredepunten", driver="GPKG") + gdf_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="geom__uittredepunten", driver="GPKG") print(BColors.OKBLUE, f"✅ Afstand en metrering tot reflijn zijn nu gekoppeld aan de uittredepunten.", BColors.ENDC) return True diff --git a/geoprob_pipe/cmd_app/spatial_joins/utils.py b/geoprob_pipe/cmd_app/spatial_joins/utils.py index e1e2f714..21dc51d8 100644 --- a/geoprob_pipe/cmd_app/spatial_joins/utils.py +++ b/geoprob_pipe/cmd_app/spatial_joins/utils.py @@ -38,13 +38,13 @@ def append_to_gis_join_parameter_invoer_table(df_sjoin: DataFrame, parameter_nam tables_names = [row[0] for row in cursor.fetchall()] # If gis_join-table is non-existent yet - if "gis_join_parameter_invoer" not in tables_names: - df_to_append.to_sql("gis_join_parameter_invoer", conn, if_exists="replace", index=False) + if "data__gis_parameter_invoer" not in tables_names: + df_to_append.to_sql("data__gis_parameter_invoer", conn, if_exists="replace", index=False) return # If polderpeil not yet in gis_join-table - df_existing = read_sql_query("SELECT * FROM gis_join_parameter_invoer;", conn) + df_existing = read_sql_query("SELECT * FROM data__gis_parameter_invoer;", conn) if parameter_name not in df_existing['parameter'].unique(): - df_to_append.to_sql("gis_join_parameter_invoer", conn, if_exists="append", index=False) + df_to_append.to_sql("data__gis_parameter_invoer", conn, if_exists="append", index=False) return raise ValueError # Should not have come here. Parameter should already be supplied, and thus user should not have been able to @@ -86,13 +86,13 @@ def append_hrd_to_gis_join_parameter_invoer_table(df_sjoin: DataFrame, geopackag tables_names = [row[0] for row in cursor.fetchall()] # If gis_join-table is non-existent yet - if "gis_join_parameter_invoer" not in tables_names: - df_to_append.to_sql("gis_join_parameter_invoer", conn, if_exists="replace", index=False) + if "data__gis_parameter_invoer" not in tables_names: + df_to_append.to_sql("data__gis_parameter_invoer", conn, if_exists="replace", index=False) return # If polderpeil not yet in gis_join-table - df_existing = read_sql_query("SELECT * FROM gis_join_parameter_invoer;", conn) + df_existing = read_sql_query("SELECT * FROM data__gis_parameter_invoer;", conn) if "polderpeil" not in df_existing['parameter'].unique(): - df_to_append.to_sql("gis_join_parameter_invoer", conn, if_exists="append", index=False) + df_to_append.to_sql("data__gis_parameter_invoer", conn, if_exists="append", index=False) return raise ValueError # Should not have come here. Parameter should already be supplied, and thus user should not have been able to diff --git a/geoprob_pipe/cmd_app/spatial_joins/coupled_exit_points_to_vakken.py b/geoprob_pipe/cmd_app/spatial_joins/vakken.py similarity index 90% rename from geoprob_pipe/cmd_app/spatial_joins/coupled_exit_points_to_vakken.py rename to geoprob_pipe/cmd_app/spatial_joins/vakken.py index 4f418e8a..194db135 100644 --- a/geoprob_pipe/cmd_app/spatial_joins/coupled_exit_points_to_vakken.py +++ b/geoprob_pipe/cmd_app/spatial_joins/vakken.py @@ -10,7 +10,7 @@ def coupled_uittredepunten_to_vakken(app_settings: ApplicationSettings) -> bool: # Read uittredepunten - gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") + gdf_exit_points: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__uittredepunten") # Check if already added if "vak_id" in gdf_exit_points.columns: @@ -18,7 +18,7 @@ def coupled_uittredepunten_to_vakken(app_settings: ApplicationSettings) -> bool: return True # Assuming already added # Spatial analyses - gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="vakindeling") + gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__vakindeling") gdf_exit_points_with_vakken = gdf_exit_points.sjoin_nearest( gdf_vakindeling[['geometry', 'id']], how='left', distance_col='distance') @@ -29,6 +29,6 @@ def coupled_uittredepunten_to_vakken(app_settings: ApplicationSettings) -> bool: gdf_new_exit_points = gdf_new_exit_points.rename(columns={"id": "vak_id"}) # Store back in geopackage - gdf_new_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="uittredepunten", driver="GPKG") + gdf_new_exit_points.to_file(Path(app_settings.geopackage_filepath), layer="geom__uittredepunten", driver="GPKG") print(BColors.OKBLUE, f"✅ Vakken zijn nu gekoppeld aan de uittredepunten.", BColors.ENDC) return True diff --git a/geoprob_pipe/cmd_app/spatial_layers/__init__.py b/geoprob_pipe/cmd_app/spatial_layers/__init__.py index 629ad723..1c5d7402 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/__init__.py +++ b/geoprob_pipe/cmd_app/spatial_layers/__init__.py @@ -1,9 +1,18 @@ -from geoprob_pipe.cmd_app.spatial_layers.dijktraject import added_dijktraject -from geoprob_pipe.cmd_app.spatial_layers.vakindeling import added_vakindeling -from geoprob_pipe.cmd_app.spatial_layers.uittredepunten.uittredepunten import added_uittredepunten -from geoprob_pipe.cmd_app.spatial_layers.hrd.hrd import added_hrd_fragility_curves -from geoprob_pipe.cmd_app.spatial_layers.polderpeil import added_polderpeil -from geoprob_pipe.cmd_app.spatial_layers.binnenteenlijn import added_binnenteenlijn -from geoprob_pipe.cmd_app.spatial_layers.buitenteenlijn import added_buitenteenlijn -from geoprob_pipe.cmd_app.spatial_layers.intredelijn import added_intredelijn -from geoprob_pipe.cmd_app.spatial_layers.ahn import added_ahn +# Deze twee moeten eerst om circulaire export te voorkomen. +from .valid_parameters import valid_parameter_list, LIST_PARAMS +from .base_inquiry import BaseInquiry + +from .added_ahn import added_ahn +from .added_binnenteenlijn import added_binnenteenlijn +from .added_buitenteenlijn import added_buitenteenlijn +from .added_dijktraject import added_dijktraject +from .added_intredelijn import added_intredelijn +from .added_parameters import added_parameters +from .added_ruimtelijke_invoer import added_ruimtelijke_input +from .added_scenarios import added_scenarios +from .added_vakindeling import added_vakindeling +from .check_batch_input import check_batch_input +from .hrd.hrd import added_hrd_fragility_curves +from .uittredepunten.uittredepunten import added_uittredepunten + + diff --git a/geoprob_pipe/cmd_app/spatial_layers/ahn.py b/geoprob_pipe/cmd_app/spatial_layers/added_ahn.py similarity index 100% rename from geoprob_pipe/cmd_app/spatial_layers/ahn.py rename to geoprob_pipe/cmd_app/spatial_layers/added_ahn.py diff --git a/geoprob_pipe/cmd_app/spatial_layers/added_binnenteenlijn.py b/geoprob_pipe/cmd_app/spatial_layers/added_binnenteenlijn.py new file mode 100644 index 00000000..691c170a --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/added_binnenteenlijn.py @@ -0,0 +1,38 @@ +""" +TODO Later Should Klein: Controleer of de binnenteenlijn ook echt aan de binnenzijde is. +TODO Later Should Klein: Niet voor elk geohydrologisch model is de binnen/buiten/intredelijn benodigd. +Maak het toevoegen van deze lijnen afhankelijk van de model keuze. +""" + +from __future__ import annotations +from typing import TYPE_CHECKING + +import fiona +from geoprob_pipe.utils.validation_messages import BColors +from .base_inquiry import BaseInquiry + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def added_binnenteenlijn(app_settings: ApplicationSettings) -> bool: + layers = fiona.listlayers(app_settings.geopackage_filepath) + + if "geom__binnenteenlijn" in layers: + print( + BColors.OKBLUE, "✔ Binnenteenlijn al toegevoegd.", BColors.ENDC + ) + return True + + request_binnenteenlijn_filepath(app_settings=app_settings) + return True + + +def request_binnenteenlijn_filepath(app_settings: ApplicationSettings): + binnenteenlijn_inquiry = BaseInquiry( + app_settings=app_settings, + param="binnenteenlijn", + specific_shape="lines", + single_geometry=True, + ) + binnenteenlijn_inquiry.request_filepath() diff --git a/geoprob_pipe/cmd_app/spatial_layers/added_buitenteenlijn.py b/geoprob_pipe/cmd_app/spatial_layers/added_buitenteenlijn.py new file mode 100644 index 00000000..835e943b --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/added_buitenteenlijn.py @@ -0,0 +1,39 @@ +""" +TODO Later Should Klein: Controleer of de buitenteenlijn ook echt aan de buitenzijde is. + +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import fiona + +from .base_inquiry import BaseInquiry +from geoprob_pipe.utils.validation_messages import BColors + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def added_buitenteenlijn(app_settings: ApplicationSettings) -> bool: + layers = fiona.listlayers(app_settings.geopackage_filepath) + + if "geom__buitenteenlijn" in layers: + print( + BColors.OKBLUE, "✔ Buitenteenlijn al toegevoegd.", BColors.ENDC + ) + return True + + request_buitenteenlijn_filepath(app_settings=app_settings) + return True + + +def request_buitenteenlijn_filepath(app_settings: ApplicationSettings): + buitenteenlijn_inquiry = BaseInquiry( + app_settings=app_settings, + param="buitenteenlijn", + specific_shape="lines", + single_geometry=True + ) + buitenteenlijn_inquiry.request_filepath() diff --git a/geoprob_pipe/cmd_app/spatial_layers/dijktraject.py b/geoprob_pipe/cmd_app/spatial_layers/added_dijktraject.py similarity index 77% rename from geoprob_pipe/cmd_app/spatial_layers/dijktraject.py rename to geoprob_pipe/cmd_app/spatial_layers/added_dijktraject.py index 18cb8a74..fe515105 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/dijktraject.py +++ b/geoprob_pipe/cmd_app/spatial_layers/added_dijktraject.py @@ -14,8 +14,8 @@ def added_dijktraject(app_settings: ApplicationSettings) -> bool: layers = fiona.listlayers(app_settings.geopackage_filepath) - if "dijktraject" in layers: - print(BColors.OKBLUE, f"✔ Dijktraject al toegevoegd.", BColors.ENDC) + if "geom__dijktraject" in layers: + print(BColors.OKBLUE, "✔ Dijktraject al toegevoegd.", BColors.ENDC) return True else: question_trajectory_source(app_settings) @@ -34,8 +34,8 @@ def question_trajectory_source(app_settings: ApplicationSettings): if choice == choices_list[0]: with importlib.resources.path( - package='geoprob_pipe.input_data.dijktrajecten', - resource='dijktrajecten.shp') as shp_path: + 'geoprob_pipe.input_data.dijktrajecten', + 'dijktrajecten.shp') as shp_path: gdf: GeoDataFrame = read_file(shp_path) specify_single_trajectory( app_settings, gdf=gdf, column_name="TRAJECT_ID") @@ -45,9 +45,8 @@ def question_trajectory_source(app_settings: ApplicationSettings): def request_trajectory_filepath(app_settings: ApplicationSettings): - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: + + while True: filepath: str = inquirer.text( message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " "referentielijn van de dijk zit.", @@ -58,12 +57,13 @@ def request_trajectory_filepath(app_settings: ApplicationSettings): if not (filepath.endswith(".gpkg") or filepath.endswith(".shp") or filepath.endswith(".gdb")): print(BColors.WARNING, f"Het bestand moet of een geopackage, shapefile of geodatabase zijn. Jouw invoer " f"eindigt op de extensie .{filepath.split(sep='.')[-1]}.", BColors.ENDC) - continue + + if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) + print(BColors.WARNING, "Het opgegeven bestandspad bestaat niet.", BColors.ENDC) continue - filepath_is_valid = True + break if filepath.endswith(".gdb"): specify_geodatabase_layer(app_settings, filepath) @@ -80,12 +80,12 @@ def request_trajectory_filepath(app_settings: ApplicationSettings): def specify_geodatabase_layer(app_settings: ApplicationSettings, filepath: str): - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: + while True: layer_name: str = inquirer.text( - message="Specificeer de layer waarin met de referentielijn van het dijktraject. Type 'listlayers' om " - "een overzicht te krijgen van de geodatabase-layers. ", + message=""" + Specificeer de layer waarin met de referentielijn van het dijktraject. Type 'listlayers' om + een overzicht te krijgen van de geodatabase-layers. Type 'cancel' om een ander bestand op te geven. + """, ).execute() layer_names = fiona.listlayers(filepath) @@ -94,13 +94,15 @@ def specify_geodatabase_layer(app_settings: ApplicationSettings, filepath: str): if layer_name == "listlayers": print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geodatabase: {layers_str}", BColors.ENDC) continue + if layer_name == "cancel": + request_trajectory_filepath(app_settings) elif layer_name not in layer_names: print(BColors.OKBLUE, f"De layer name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " f"de geodatabase: {layers_str}", BColors.ENDC) continue # TODO Later Must Klein: Check dat een LineString-laag wordt opgegeven. - layer_name_is_valid = True + break with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") @@ -109,12 +111,12 @@ def specify_geodatabase_layer(app_settings: ApplicationSettings, filepath: str): def specify_geopackage_layer(app_settings: ApplicationSettings, filepath: str): - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: + while True: layer_name: str = inquirer.text( - message="Specificeer de layer waarin met de referentielijn van het dijktraject. Type 'listlayers' om " - "een overzicht te krijgen van de geopackage-layers. ", + message=""" + Specificeer de layer waarin met de referentielijn van het dijktraject. Type 'listlayers' om + een overzicht te krijgen van de geodatabase-layers. Type 'cancel' om een ander bestand op te geven. + """, ).execute() layer_names = fiona.listlayers(filepath) @@ -123,25 +125,29 @@ def specify_geopackage_layer(app_settings: ApplicationSettings, filepath: str): if layer_name == "listlayers": print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geopackage: {layers_str}", BColors.ENDC) continue + if layer_name == "cancel": + request_trajectory_filepath(app_settings) + elif layer_name not in layer_names: print(BColors.OKBLUE, f"De layer name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " f"de geopackage: {layers_str}", BColors.ENDC) continue # TODO Later Must Klein: Check dat een LineString-laag wordt opgegeven. - layer_name_is_valid = True + break gdf: GeoDataFrame = read_file(filepath, layer=layer_name) specify_column_with_trajectory_name(app_settings, gdf) def specify_column_with_trajectory_name(app_settings: ApplicationSettings, gdf: GeoDataFrame): - column_name: Optional[str] = None - column_name_is_valid = False - while column_name_is_valid is False: + + while True: column_name: str = inquirer.text( - message="Specificeer de kolom waarin de naam van het dijktraject staat. Type 'listcolumns' om " - "een overzicht te krijgen van de kolommen. ", + message=""" + Specificeer de kolom waarin de naam van het dijktraject staat. Type 'listcolumns' om + een overzicht te krijgen van de kolommen. Type 'cancel' om een ander bestand op te geven. + """, ).execute() column_names = gdf.columns @@ -150,12 +156,14 @@ def specify_column_with_trajectory_name(app_settings: ApplicationSettings, gdf: print(BColors.OKBLUE, f"De volgende kolommen zijn beschikbaar in de spatial layer: {columns_str}", BColors.ENDC) continue + if column_name == "cancel": + request_trajectory_filepath(app_settings) elif column_name not in column_names: print(BColors.OKBLUE, f"De kolom naam '{column_name}' bestaat niet. De volgende kolommen zijn beschikbaar " f"in de spatial layer: {columns_str}", BColors.ENDC) continue - column_name_is_valid = True + break column_name: str specify_single_trajectory(app_settings, gdf, column_name) @@ -170,21 +178,20 @@ def specify_single_trajectory(app_settings: ApplicationSettings, gdf: GeoDataFra column_name: "traject_naam", gdf.geometry.name: "geometry", }) - gdf.to_file(app_settings.geopackage_filepath, layer="dijktraject", driver="GPKG") + gdf.to_file(app_settings.geopackage_filepath, layer="geom__dijktraject", driver="GPKG") - print(BColors.OKBLUE, f"✅ Trajectlijn toegevoegd.", BColors.ENDC) + print(BColors.OKBLUE, "✅ Trajectlijn toegevoegd.", BColors.ENDC) return # Multiple items in gdf? Make user select single one - trajectory_name: Optional[str] = None - trajectory_name_is_valid = False - while trajectory_name_is_valid is False: + + while True: trajectory_name: str = inquirer.text( message=f"Er zijn {gdf.__len__()} opties. Type hier welke de juiste referentielijn is. Type " f"'listoptions' om een overzicht te krijgen van de opties.", ).execute() - trajectory_names = gdf[column_name].values.tolist() + trajectory_names = gdf[column_name].values.astype(str).tolist() trajectory_names.sort() trajectories_str = ", ".join(trajectory_names) if trajectory_name == "listoptions": @@ -196,7 +203,7 @@ def specify_single_trajectory(app_settings: ApplicationSettings, gdf: GeoDataFra f"{trajectories_str}", BColors.ENDC) continue - trajectory_name_is_valid = True + break gdf = gdf[gdf[column_name] == trajectory_name] gdf = gdf[[column_name, gdf.geometry.name]] @@ -204,6 +211,6 @@ def specify_single_trajectory(app_settings: ApplicationSettings, gdf: GeoDataFra column_name: "traject_naam", gdf.geometry.name: "geometry", }) - gdf.to_file(app_settings.geopackage_filepath, layer="dijktraject", driver="GPKG") - print(BColors.OKBLUE, f"✅ Trajectlijn toegevoegd.", BColors.ENDC) + gdf.to_file(app_settings.geopackage_filepath, layer="geom__dijktraject", driver="GPKG") + print(BColors.OKBLUE, "✅ Trajectlijn toegevoegd.", BColors.ENDC) return diff --git a/geoprob_pipe/cmd_app/spatial_layers/added_intredelijn.py b/geoprob_pipe/cmd_app/spatial_layers/added_intredelijn.py new file mode 100644 index 00000000..c4745b75 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/added_intredelijn.py @@ -0,0 +1,36 @@ +""" +TODO Later Should Klein: Controleer of de intredelijn ook echt aan de binnenzijde is. + +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import fiona + +from .base_inquiry import BaseInquiry +from geoprob_pipe.utils.validation_messages import BColors + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def added_intredelijn(app_settings: ApplicationSettings) -> bool: + layers: list[str] = fiona.listlayers(app_settings.geopackage_filepath) + for layer in layers: + if "intredelijn" in layer.split("__"): + print(BColors.OKBLUE, "✔ Intredelijn al toegevoegd.", BColors.ENDC) + return True + + request_intredelijn_filepath(app_settings=app_settings) + return True + + +def request_intredelijn_filepath(app_settings: ApplicationSettings): + intredelijn_inquiry = BaseInquiry( + app_settings=app_settings, + param="intredelijn", + specific_shape="lines", + ) + intredelijn_inquiry.request_filepath() diff --git a/geoprob_pipe/cmd_app/spatial_layers/added_parameters.py b/geoprob_pipe/cmd_app/spatial_layers/added_parameters.py new file mode 100644 index 00000000..bd6df504 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/added_parameters.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +import InquirerPy.prompts.input as prompt + +from geoprob_pipe.utils.validation_messages import BColors + +from .valid_parameters import valid_parameter_list + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def added_parameters(app_settings: ApplicationSettings): + """ + Check of de parameters voor ruimtelijke invoer al zijn toegevoegd aan de metadata. + + :param app_settings: `ApplicationSettings` object. + :return: bool + """ + conn = sqlite3.connect(app_settings.geopackage_filepath) + cur = conn.cursor() + cur.execute( + "SELECT EXISTS (SELECT 1 FROM geoprob_pipe_metadata WHERE metadata_type = ?)", + ("ruimtelijke_parameters",), + ) + check = cur.fetchone()[0] == 1 + if not check: + request_parameters(app_settings) + return True + + +def request_parameters(app_settings: ApplicationSettings): + """ + Method voor het opvragen van de lijst met parameters waarvoor ruimtelijke + invoer gewenst is. Er wordt gecheckt of deze parameters ondersteund worden + en of deze in het gekozen model zitten. + + :param app_settings: `ApplicationSettings` object. + """ + conn = sqlite3.connect(app_settings.geopackage_filepath) + + valid_list = valid_parameter_list(app_settings) + + parameters: list[str] = [] + + while True: + parameter_input: str = prompt.InputPrompt( + message=( + """ +Specificeer welke parameters je wilt toevoegen als ruimtelijke input. +Doe dit door de namen gescheiden met comma's op te geven. +Type 'list' om de lijst te zien met alle parameters die invoerbaar zijn voor dit model. +Als je geen ruimtelijke input wilt ingeven druk dan op enter zonder iets in te vullen. +""" + ) + ).execute() + if parameter_input == "": + parameters = [] + elif parameter_input == "list": + params_str = ", ".join(valid_list) # type:ignore + print( + BColors.OKBLUE, + f"De volgende parameters zijn beschikbaar voor ruimtelijke input: {params_str}", + BColors.ENDC, + ) + continue + + else: + parameters = parameter_input.split(",") + + if not all(param in valid_list for param in parameters): + print( + BColors.WARNING, + "Een of meerdere van de opgegeven parameters zijn niet geschikt voor ruimtelijke invoer.", + BColors.ENDC, + ) + continue + break + + sql_update = "UPDATE geoprob_pipe_metadata SET metadata_value = ? WHERE metadata_type = ?" + sql_insert = "INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES (?, ?)" + + with conn: # transaction + cur = conn.execute( + sql_update, (", ".join(parameters), "ruimtelijke_parameters") + ) + if cur.rowcount == 0: + conn.execute( + sql_insert, ("ruimtelijke_parameters", ", ".join(parameters)) + ) + + conn.commit() + conn.close() diff --git a/geoprob_pipe/cmd_app/spatial_layers/added_ruimtelijke_invoer.py b/geoprob_pipe/cmd_app/spatial_layers/added_ruimtelijke_invoer.py new file mode 100644 index 00000000..1803f7c6 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/added_ruimtelijke_invoer.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +import fiona + +from geoprob_pipe.cmd_app.spatial_layers import LIST_PARAMS, BaseInquiry +from geoprob_pipe.utils.validation_messages import BColors + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def added_ruimtelijke_input(app_settings: ApplicationSettings) -> bool: + conn = sqlite3.connect(app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_parameters",), + ) + parameters: list[str] = cursor.fetchone()[0].split(", ") + conn.close() + + list_layers: list[str] = fiona.listlayers(app_settings.geopackage_filepath) + + # De split maakt één lege string als de uit de metadata gelezen string leeg is. + if parameters == [""]: + print( + BColors.OKBLUE, + "✔ Geen ruimtelijke parameter invoer.", + BColors.ENDC, + ) + return True + + for parameter in parameters: + if any(parameter in layer.split("__") for layer in list_layers): + print( + BColors.OKBLUE, + f"✔ {parameter} al toegevoegd.", + BColors.ENDC, + ) + continue + + inquiry = BaseInquiry( + app_settings=app_settings, + param=f"{parameter}", + specific_shape=LIST_PARAMS[parameter]["shape"], + include_value=True, + ) + inquiry.request_filepath() + + return True diff --git a/geoprob_pipe/cmd_app/spatial_layers/added_scenarios.py b/geoprob_pipe/cmd_app/spatial_layers/added_scenarios.py new file mode 100644 index 00000000..7e622c4d --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/added_scenarios.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +from InquirerPy.prompts.input import InputPrompt + +from geoprob_pipe.utils.validation_messages import BColors + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def added_scenarios(app_settings: ApplicationSettings): + """ + Check of de scenarios voor ruimtelijke invoer al zijn toegevoegd aan de metadata. + + :param app_settings: `ApplicationSettings` object. + :return: bool + """ + conn = sqlite3.connect(app_settings.geopackage_filepath) + cur = conn.cursor() + cur.execute( + "SELECT EXISTS (SELECT 1 FROM geoprob_pipe_metadata WHERE metadata_type = ?)", + ("ruimtelijke_scenarios",), + ) + check = cur.fetchone()[0] == 1 + if not check: + request_scenarios(app_settings) + return True + + +def request_scenarios(app_settings: ApplicationSettings): + """ + Vraag de mogelijke lijst van scenarios voor de ruimtelijke invoer op en voeg + deze toe aan de metadata. + + :param app_settings: `ApplicationSettings` object. + """ + + while True: + scenario_input: str = InputPrompt( + message=""" +Specificeer welke scenarios je wilt toevoegen als ruimtelijke input. +Doe dit door de namen gescheiden met comma's op te geven. Bijvoorbeeld: Scenario1, Scenario2. +De scenario namen moeten als kolom in de attributes van de laag staan. +Als je geen ruimtelijke input wilt ingeven druk dan op enter zonder iets in te vullen. +""" + ).execute() + if scenario_input == "": + scenarios = "" + else: + try: + scenarios = scenario_input.split(",") + scenarios = ", ".join(scenarios) + except Exception: + print( + BColors.OKBLUE, + f"Geen scenarios toegevoegd. '{scenario_input}' is geen geldige invoer.", + BColors.ENDC, + ) + continue + break + + conn = sqlite3.connect(app_settings.geopackage_filepath) + + sql_update = "UPDATE geoprob_pipe_metadata SET metadata_value = ? WHERE metadata_type = ?" + sql_insert = "INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES (?, ?)" + + with conn: # transaction + cur = conn.execute(sql_update, (scenarios, "ruimtelijke_scenarios")) + if cur.rowcount == 0: + conn.execute(sql_insert, ("ruimtelijke_scenarios", scenarios)) + + conn.commit() + conn.close() diff --git a/geoprob_pipe/cmd_app/spatial_layers/vakindeling.py b/geoprob_pipe/cmd_app/spatial_layers/added_vakindeling.py similarity index 70% rename from geoprob_pipe/cmd_app/spatial_layers/vakindeling.py rename to geoprob_pipe/cmd_app/spatial_layers/added_vakindeling.py index eb40641c..9a522438 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/vakindeling.py +++ b/geoprob_pipe/cmd_app/spatial_layers/added_vakindeling.py @@ -13,13 +13,14 @@ from geopandas import GeoDataFrame import fiona from geoprob_pipe.utils.validation_messages import BColors + if TYPE_CHECKING: from geoprob_pipe.cmd_app.cmd import ApplicationSettings def added_vakindeling(app_settings: ApplicationSettings) -> bool: layers = fiona.listlayers(app_settings.geopackage_filepath) - if "vakindeling" in layers: + if "geom__vakindeling" in layers: check_validity_vakindeling(app_settings=app_settings) return True else: @@ -28,7 +29,7 @@ def added_vakindeling(app_settings: ApplicationSettings) -> bool: def check_validity_vakindeling(app_settings: ApplicationSettings): - gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="dijktraject") + gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__dijktraject") gdf_dijktraject_geom = gdf_dijktraject.iloc[0].geometry if isinstance(gdf_dijktraject_geom, MultiLineString): assert gdf_dijktraject_geom.geoms.__len__() == 1 @@ -40,22 +41,28 @@ def check_validity_vakindeling(app_settings: ApplicationSettings): f"developer.'") dijktraject_length = round(ls_dijktraject.length, 2) - gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="vakindeling") + gdf_vakindeling: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__vakindeling") vakindeling_geometries = gdf_vakindeling.geometry.tolist() vakindeling_total_length = round(sum([geom.length for geom in vakindeling_geometries]), 2) assert dijktraject_length == vakindeling_total_length - print(BColors.OKBLUE, f"✔ Vakindeling al toegevoegd.", BColors.ENDC) - - -def import_from_geopackage(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de laag met de vakindeling. " - "Type 'listlayers' om een overzicht te krijgen van de geopackage-layers. ", - ).execute() + print(BColors.OKBLUE, "✔ Vakindeling al toegevoegd.", BColors.ENDC) + + +def import_from_geopackage(app_settings: ApplicationSettings, + filepath: str) -> GeoDataFrame: + layer_name: str = "" + skip_batch = False # Als batch input faalt ga over op handmatig + while True: + if app_settings.batch_input and not skip_batch: + layer_name = app_settings.input_config.get( + "vakindeling", "database_layer" + ) + else: + layer_name: str = inquirer.text( + message="Specificeer de laag met de vakindeling. " + "Type 'listlayers' om een overzicht te krijgen van de geopackage-layers. ", + ).execute() layer_names = fiona.listlayers(filepath) layer_names.sort() @@ -66,9 +73,10 @@ def import_from_geopackage(filepath: str) -> GeoDataFrame: elif layer_name not in layer_names: print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " f"de geopackage: {layers_str}", BColors.ENDC) + skip_batch = True continue - layer_name_is_valid = True + break with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") @@ -76,14 +84,21 @@ def import_from_geopackage(filepath: str) -> GeoDataFrame: return gdf -def import_from_geodatabase(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de laag met de vakindeling. " - "Type 'listlayers' om een overzicht te krijgen van de geodatabase-layers. ", - ).execute() +def import_from_geodatabase(app_settings: ApplicationSettings, + filepath: str) -> GeoDataFrame: + layer_name: str = "" + skip_batch = False # Als batch input faalt ga over op handmatig + + while True: + if app_settings.batch_input and not skip_batch: + layer_name = app_settings.input_config.get( + "vakindeling", "database_layer" + ) + else: + layer_name: str = inquirer.text( + message="Specificeer de laag met de vakindeling. " + "Type 'listlayers' om een overzicht te krijgen van de geodatabase-layers. ", + ).execute() layer_names = fiona.listlayers(filepath) layer_names.sort() @@ -94,9 +109,10 @@ def import_from_geodatabase(filepath: str) -> GeoDataFrame: elif layer_name not in layer_names: print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " f"de geodatabase: {layers_str}", BColors.ENDC) + skip_batch = True continue - layer_name_is_valid = True + break with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") @@ -105,13 +121,17 @@ def import_from_geodatabase(filepath: str) -> GeoDataFrame: def request_vakindeling_filepath(app_settings: ApplicationSettings): - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: - filepath: str = inquirer.text( - message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " - "vakindeling van de dijk zit.", - ).execute() + filepath: str = "" + skip_batch = False # Als batch input faalt ga over op handmatig + + while True: + if app_settings.batch_input and not skip_batch: + filepath = app_settings.input_config.get("vakindeling", "filepath") + else: + filepath: str = inquirer.text( + message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " + "vakindeling van de dijk zit.", + ).execute() filepath = filepath.replace('"', '') @@ -120,10 +140,11 @@ def request_vakindeling_filepath(app_settings: ApplicationSettings): f"eindigt op de extensie .{filepath.split(sep='.')[-1]}.", BColors.ENDC) continue if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) + print(BColors.WARNING, "Het opgegeven bestandspad bestaat niet.", BColors.ENDC) + skip_batch = True continue - filepath_is_valid = True + break if filepath.endswith(".shp"): with warnings.catch_warnings(): @@ -131,10 +152,12 @@ def request_vakindeling_filepath(app_settings: ApplicationSettings): gdf: GeoDataFrame = read_file(filepath) validate_vakindeling(app_settings, gdf=gdf) elif filepath.endswith(".gpkg"): - gdf: GeoDataFrame = import_from_geopackage(filepath=filepath) + gdf: GeoDataFrame = import_from_geopackage(app_settings=app_settings, + filepath=filepath) validate_vakindeling(app_settings, gdf=gdf) elif filepath.endswith(".gdb"): - gdf: GeoDataFrame = import_from_geodatabase(filepath=filepath) + gdf: GeoDataFrame = import_from_geodatabase(app_settings=app_settings, + filepath=filepath) validate_vakindeling(app_settings, gdf=gdf) else: raise NotImplementedError(f"File with extension {filepath.split(sep='.')[-1]} is not yet supported. " @@ -153,14 +176,20 @@ def validate_vakindeling(app_settings: ApplicationSettings, gdf: GeoDataFrame): def specify_column_with_vaknaam( app_settings: ApplicationSettings, gdf: GeoDataFrame): - column_name: Optional[str] = None - column_name_is_valid = False - while column_name_is_valid is False: - column_name: str = inquirer.text( - message="Specificeer de kolom waarin de vaknaam staat. Type " - "'listcolumns' om een overzicht te krijgen van de " - "kolommen. ", - ).execute() + column_name: str = "" + + skip_batch = False # Als batch input faalt ga over op handmatig + while True: + if app_settings.batch_input and not skip_batch: + column_name = app_settings.input_config.get( + "vakindeling", "vak_naam_kolom" + ) + else: + column_name: str = inquirer.text( + message="Specificeer de kolom waarin de vaknaam staat. Type " + "'listcolumns' om een overzicht te krijgen van de " + "kolommen. ", + ).execute() column_names = gdf.columns columns_str = ", ".join(column_names) @@ -174,9 +203,10 @@ def specify_column_with_vaknaam( f"De kolom naam '{column_name}' bestaat niet. De volgende " f"kolommen zijn beschikbaar in de spatial layer: " f"{columns_str}", BColors.ENDC) + skip_batch = True continue - column_name_is_valid = True + break column_name: str specify_column_with_vak_id( @@ -193,14 +223,20 @@ def is_numeric_integer(val): def specify_column_with_vak_id( app_settings: ApplicationSettings, gdf: GeoDataFrame, kolom_vak_naam: str): - kolom_vak_id: Optional[str] = None - column_name_is_valid = False - while column_name_is_valid is False: - kolom_vak_id: str = inquirer.text( - message="Specificeer de kolom waarin het vak id staat. Indien " - "onnodig, type 'nvt'. Type 'listcolumns' om een overzicht " - "te krijgen van de kolommen. ", - ).execute() + kolom_vak_id: str = "" + skip_batch = False # Als batch input faalt ga over op handmatig + + while True: + if app_settings.batch_input and not skip_batch: + kolom_vak_id = app_settings.input_config.get( + "vakindeling", "vak_id_kolom" + ) + else: + kolom_vak_id: str = inquirer.text( + message="Specificeer de kolom waarin het vak id staat. Indien " + "onnodig, type 'nvt'. Type 'listcolumns' om een overzicht " + "te krijgen van de kolommen. ", + ).execute() column_names = gdf.columns columns_str = ", ".join(column_names) @@ -218,6 +254,7 @@ def specify_column_with_vak_id( print(f"{BColors.OKBLUE}De kolom naam '{kolom_vak_id}' bestaat " f"niet. De volgende kolommen zijn beschikbaar in de spatial " f"layer: {columns_str}{BColors.ENDC}") + skip_batch = True continue # Ensure column values are unique and integers @@ -225,15 +262,17 @@ def specify_column_with_vak_id( print(f"{BColors.OKBLUE}De waarden in deze kolom zijn niet uniek. " f"Corrigeer de dubbelingen, of kies een andere kolom." f"{BColors.ENDC}") + skip_batch = True continue elif not gdf[kolom_vak_id].apply(is_numeric_integer).all(): print(f"{BColors.OKBLUE}De waarden in deze kolom zijn niet allen " f"volledige getallen (integers). Corrigeer de kolom, of " f"kies een andere.{BColors.ENDC}") + skip_batch = True continue - column_name_is_valid = True + break kolom_vak_id: str align_vak_shp_to_dijktraject( @@ -250,7 +289,7 @@ def align_vak_shp_to_dijktraject( # Data verzamelen uit provided vak shp rows = [] - for index, row in gdf_vakindeling.iterrows(): + for _, row in gdf_vakindeling.iterrows(): pnt1 = row.geometry.boundary.geoms[0] pnt2 = row.geometry.boundary.geoms[1] m_pnt1 = round(ls_dijktraject.project(pnt1), 1) @@ -270,7 +309,7 @@ def align_vak_shp_to_dijktraject( # Align vak geometry to dijktraject by retrieving substring rows = [] - for index, row in df.iterrows(): + for _, row in df.iterrows(): new_row = { "naam": row["naam"], "m_start": row["m_start"], @@ -289,5 +328,5 @@ def align_vak_shp_to_dijktraject( # Add to geopackage gdf_new_vakindeling.to_file( Path(app_settings.geopackage_filepath), - layer="vakindeling", driver="GPKG") - print(BColors.OKBLUE, f"✅ Vakindeling toegevoegd.", BColors.ENDC) + layer="geom__vakindeling", driver="GPKG") + print(BColors.OKBLUE, "✅ Vakindeling toegevoegd.", BColors.ENDC) diff --git a/geoprob_pipe/cmd_app/spatial_layers/base_inquiry.py b/geoprob_pipe/cmd_app/spatial_layers/base_inquiry.py new file mode 100644 index 00000000..b3792151 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/base_inquiry.py @@ -0,0 +1,581 @@ +""" +Class voor het opvragen van de ruimtelijke invoer voor de parameters +vanuit GIS lagen. Shapefiles, geodatabases en geopackages worden ondersteund. + +Er zijn vier mogelijke invoer situaties: +-GEEN ruimtelijke scenarios en GEEN ruimtelijke data invoer +-GEEN ruimtelijke scenarios en WEL ruimtelijke data invoer +-WEL ruimtelijke scenarios en GEEN ruimtelijke data invoer +-WEL ruimtelijke scenarios en WEL ruimtelijke data invoer + +Daarnaast is er nog onderscheid in invoer met verschillende shapes per scenario +of twee sets met invoer data. +""" + +from __future__ import annotations + +import os +import sqlite3 +import warnings +from typing import TYPE_CHECKING + +import fiona +import geopandas as gpd +import InquirerPy.prompts.input as prompt +import pandas as pd +from shapely import LineString, MultiLineString + +from geoprob_pipe.utils.validation_messages import BColors + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +class BaseInquiry: + def __init__( + self, + app_settings: ApplicationSettings, + param: str, + specific_shape: str = "", + include_value: bool = False, + single_geometry: bool = False, + ) -> None: + """ + Class voor de opvraag van de ruimtelijke invoer van de parameters. + + :param app_settings: `ApplicationSettings` object. + :param param: Parameter die wordt opgevraagd. Deze wordt ook gebruikt + voor de naam van de tabel in de geopackage. + :param specific_shape: Welke soort ruimtelijke invoer acceptabel is, defaults to "". + Geldige invoer is "lines", "polygons" of "rasters". + :param include_value: Of er naast de geometrie zelf ook een bepaalde + waarde moet worden toegevoegd aan de geopackage, defaults to False + """ + self.app_settings = app_settings + self.param = param + self.specific_shape = specific_shape + self.include_value = include_value + self.single_geometry = single_geometry + conn = sqlite3.connect(app_settings.geopackage_filepath) + cur = conn.cursor() + cur.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_scenarios",), + ) + self.scenarios: list[str] = cur.fetchone()[0].split(", ") + conn.close() + + def request_filepath(self): + """ + Method voor het opvragen van het filepath. + """ + skip_batch = False # Als batch input faalt ga over op handmatig + while True: + if self.app_settings.batch_input and not skip_batch: + filepath = self.app_settings.input_config.get( + f"{self.param}", "filepath" + ) + else: + filepath: str = prompt.InputPrompt( + message=( + f""" +Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase +met de {self.param} geometrieën. +""" + ) + ).execute() + + filepath = filepath.replace('"', "") + + if not os.path.exists(filepath): + print( + BColors.WARNING, + "Het opgegeven bestandspad bestaat niet.", + BColors.ENDC, + ) + skip_batch = True + continue + + if not ( + filepath.endswith(".gpkg") + or filepath.endswith(".shp") + or filepath.endswith(".gdb") + ): + print( + BColors.WARNING, + f""" + Het bestand moet of een geopackage, shapefile of geodatabase zijn. + Jouw invoer eindigt op de extensie .{filepath.split(sep=".")[-1]}. + """, + BColors.ENDC, + ) + skip_batch = True + continue + + # Import data + gdf = self._import_data(filepath) # type:ignore + + # Add data to geopackage + if self._add_to_gpkg(gdf): + break + + print(BColors.OKBLUE, f"✅ {self.param} toegevoegd.", BColors.ENDC) + + def _import_data(self, filepath: str) -> gpd.GeoDataFrame: + """ + Helper method voor het importeren van de data. + + :param filepath: Bestandspad van de shapefile of database met de GIS lagen. + :raises NotImplementedError: Wanneer het bestand iets anders is dan wordt ondersteund. + :return: GeoDataFrame met de uitgelezen data. + :rtype GeoDataFrame: + """ + if filepath.endswith(".shp"): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Measured \\(M\\) geometry types are not supported.*", + ) + gdf: gpd.GeoDataFrame = gpd.read_file(filepath) + + elif filepath.endswith(".gpkg"): + gdf: gpd.GeoDataFrame = self._import_from_db(filepath) + + elif filepath.endswith(".gdb"): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Measured \\(M\\) geometry types are not supported.*", + ) + gdf: gpd.GeoDataFrame = self._import_from_db(filepath) + + else: + raise NotImplementedError( + f"Applicatie vroegtijdig afgesloten: Een {filepath.split(sep='.')[-1]}-bestand is niet geïmplementeerd." + ) + + return gdf + + def _import_from_db(self, filepath: str) -> gpd.GeoDataFrame: + """ + Helper method voor het importeren vanuit een gbd of een geopackage. + + :param filepath: Bestandspad van de database. + :return: GeoDataFrame met de uitgelezen data. + """ + skip_batch = False # Als batch input faalt ga over op handmatig + while True: + if self.app_settings.batch_input and not skip_batch: + layer_name = self.app_settings.input_config.get( + f"{self.param}", "database_layer" + ) + else: + layer_name: str = prompt.InputPrompt( + message=( + f""" +Specificeer de layer waarin de {self.param} staat. Type 'listlayers' om +een overzicht te krijgen van de geodatabase-layers. Type 'cancel' om een +ander bestand op te gaven. +""" + ) + ).execute() + + layer_names = fiona.listlayers(filepath) + layer_names.sort() + layers_str = ", ".join(layer_names) + if layer_name == "listlayers": + print( + f"{BColors.OKBLUE}De volgende layers zijn beschikbaar in de geodatabase: {layers_str}{BColors.ENDC}" + ) + continue + + if layer_name == "cancel": + self.request_filepath() + + elif layer_name not in layer_names: + print( + f"{BColors.OKBLUE} De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " + f"de geodatabase: {layers_str}{BColors.ENDC}" + ) + skip_batch = True + continue + + break + + gdf: gpd.GeoDataFrame = gpd.read_file(filepath, layer=layer_name) # type:ignore + return gdf + + def _import_from_shp(self, filepath: str) -> gpd.GeoDataFrame: + """ + Helper method voor het importeren vanuit een shapefile. + + :param filepath: Bestandspad van de shapefile. + :return: GeoDataFrame met de uitgelezen data. + """ + + gdf: gpd.GeoDataFrame = gpd.read_file(filepath) + return gdf + + def _check_shape(self, gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame | None: + """ + Helper method voor het checken of de geometry het juiste type is. + + :param gdf: GeoDataFrame met de uitgelezen data. + :return: GeoDataFrame of None als check faalt. + """ + + if self.specific_shape == "": + return gdf + + if self.specific_shape == "lines": + all_lines = gdf.geometry.apply( + lambda geom: ( + isinstance(geom, LineString) + or isinstance(geom, MultiLineString) + ) + ).all() + + if not all_lines: + print( + BColors.WARNING, + """ + Het geïmporteerde bestand bestaat niet (volledig) uit lijnen, + maar ook uit andere typen geometrie. Enkel lijnen zijn toegestaan. + """, + BColors.ENDC, + ) + self.request_filepath() + + return gdf + + def _add_to_gpkg(self, gdf: gpd.GeoDataFrame) -> bool: + """ + Helper method om de data aan de geopackage toe te voegen. + Start een apart inquiry als en waarden moeten worden toegevoegd aan de + shape of als er meerdere shapes moeten worden toegevoegd. De kolommen + moeten van te voren met de juiste suffix in de shape staan. Als scheiding + wordt een dubble underscore gebruikt. + + De mogelijke suffixen zijn: + *__mean voor het gemiddelde of deterministische waarde. + *__dist voor het distributie type. + *__var voor de variatie. + *__dev voor de deviatie. + *__min voor het minimum. + *__max voor het maximum. + + Als er meerdere scenarios zijn moeten deze als aparte kolommen worden + toegevoegd met een prefix. Bijvoorbeeld scenario1__*. + + Of als de scenarios als aparte shapes worden ingevoerd, moet er een kolom + met de naam "ondergrondscenario" zijn toegevoegd. + + :param gdf: GeoDataFrame met de uitgelezen data van de laag. + :return: bool voor valid_input in while-loop. + :rtype: bool + """ + self.suffix_list = ["mean", "dist", "var", "dev", "min", "max"] + + if self.include_value and self.scenarios == "": + # Case: Alleen suffix, alle kolommen worden met de geometrie weggeschreven. + return self._add_with_suffix(gdf) + + elif self.include_value and self.scenarios != "": + # Case: beide affixes, per scenario worden alle kolommen met de + # geometrieën weggeschreven. + # Of verschillende geometrieën per scenario + column_list = gdf.columns.to_list() + if True in [ + len(column.split("__")) == 3 + and self.param in column.split("__") + for column in column_list + ]: + return self._add_with_affixes(gdf) + else: + return self._add_separate_with_suffix(gdf) + + elif ( + not self.include_value + and ( + self.scenarios == "" # Geen scenarios opgegeven. + or self.single_geometry # Vang hier als scenario's niet relevant zijn. + or not self._scenario_check( + gdf.columns.to_list() + ) # Scenario's niet gekoppeld aan deze laag. + ) + ): + # Case: geen affixes alleen de geometrie wordt geschreven. + gdf_to_add = gdf[["geometry"]] + gdf_to_add.to_file( + self.app_settings.geopackage_filepath, + layer=f"geom__{self.param}", + driver="GPKG", + ) + return True + + elif not self.include_value and self.scenarios != "": + # Case: Alleen losse shapes, de geometrieën worden per scenario geschreven. + return self._add_with_separate_geometry(gdf) + + else: + raise ImportError( + "De laag kan niet worden verwerkt. Controleer of de juiste kolommen aanwezig zijn." + ) + + # Methods voor het controleren van de geodataframes. + def _scenario_check(self, column_list: list[str]): + """ + Check of het scenario in de 'ondergrondscenario' kolom voorkomt. + + :param list[str] column_list: Lijst met kolomnamen uit de GeoDataFrame. + :return: Check of het scenario voorkomt. + :rtype: bool + """ + if "ondergrondscenario" not in column_list: + print( + BColors.WARNING, + "Geen kolom met de naam 'ondergrondscenario' gevonden.\n", + "De volgende kolommen zijn gevonden:\n", + f"{column_list}", + BColors.ENDC, + ) + + return False + else: + return True + + def _distribution_check( + self, + column_list: list[str], + add_list: list[str], + gdf: gpd.GeoDataFrame, + scenario: str = "", + ) -> bool: + """ + Check of de kolommen de juiste inputs hebben voor de gekozen distributie. + Als een var of std nodig is, deze aanwezig is. En niet beide aanwezig zijn. + + :param list[sts] column_list: Lijst met kolomnamen. + :param list[str] add_list: Lijst met geselecteerde kolomnamen voor toe te voegen. + :param gpd.GeaDataFrame gdf: GeoDataFrame van uitgelezen data. + :return: Check of de kolommen goed zijn. + :rtype: bool + """ + if len(add_list) == 0: + print( + BColors.WARNING, + "De parameter is niet gevonden in de attributen van de laag.", + f"Alleen deze kolommen zijn gevonden: {', '.join(column_list)}", + BColors.ENDC, + ) + return False + + # Check var or std if not deterministic. + for _, row in gdf.iterrows(): + # Als per scenario wordt gecontroleerd is de naam in de gdf anders. + if scenario != "": + prefix = scenario + "__" + else: + prefix = "" + + if row[f"{prefix}{self.param}__dist"] == "deterministic": + continue # Overslaan + + if f"{self.param}__var" in add_list: + value = float(row[f"{prefix}{self.param}__var"]) + var_bool = pd.notna(value) + else: + var_bool = False + + if f"{self.param}__dev" in add_list: + value = float(row[f"{prefix}{self.param}__dev"]) + dev_bool = pd.notna(value) + else: + dev_bool = False + + if var_bool and dev_bool: + print( + BColors.WARNING, + "Er is zowel een waarde voor de variatie als de standaard deviatie.", + BColors.ENDC, + ) + return False + + if not var_bool and not dev_bool: + print( + BColors.WARNING, + "Er is geen waarde voor de variatie of de standaard deviatie.", + BColors.ENDC, + ) + return False + + return True + + # Methods voor het toevoegen aan de database + + def _add_with_separate_geometry(self, gdf: gpd.GeoDataFrame) -> bool: + """ + Method voor het toevoegen van lagen met meerdere shapes voor + verschillende scenarios. + + :param gdf: GeoDataFrame met de uitgelezen data. + :return: bool voor valid_input in while-loop. + :rtype: bool + """ + column_list = gdf.columns.to_list() + if not self._scenario_check(column_list): + return False + + for scenario in self.scenarios: + mask = gdf["ondergrondscenario"] == scenario + gdf_to_add = gdf[mask][["geometry"]] + + if len(gdf_to_add) == 0: + print( + BColors.WARNING, + f"Geen geometrieën gevonden met de ondergrondscenarionaam: {scenario}", + BColors.ENDC, + ) + continue + + gdf_to_add.to_file( + self.app_settings.geopackage_filepath, + layer=f"geom__{self.param}__{scenario}", + driver="GPKG", + ) + + return True + + def _add_with_suffix(self, gdf: gpd.GeoDataFrame) -> bool: + """ + Method voor het schrijven naar de geopackage voor een case met + parameter invoer en geen scenarios. + + :param gdf: GeoDataFrame met de uitgelezen data. + :return: bool voor valid_input in while-loop. + :rtype: bool + """ + column_list = gdf.columns.to_list() + add_list = [ + f"{self.param}__{param}" + for param in self.suffix_list + if f"{self.param}__{param}" in column_list + ] + + if not self._distribution_check(column_list, add_list, gdf): + return False + + gdf_to_add = gdf[["geometry", add_list]] + + gdf_to_add.to_file( + self.app_settings.geopackage_filepath, + layer=f"geom__{self.param}", + driver="GPKG", + ) + + return True + + def _add_with_affixes(self, gdf: gpd.GeoDataFrame) -> bool: + """ + Voeg de data toe apart per scenario als deze als aparte kolommen in de + laag staan. + + :param gdf: GeoDataFrame met de uitgelezen data. + :return: bool voor valid_input in while-loop + :rtype: bool + """ + column_list = gdf.columns.to_list() + + for scenario in self.scenarios: + filter_list = [ + column + for column in column_list + if column.split("__")[0] == scenario + ] + + if len(filter_list) == 0: + print( + BColors.WARNING, + f"Geen geometrieën gevonden met de ondergrondscenarionaam: {scenario}", + BColors.ENDC, + ) + continue + + # Strip scenario van kolomnaam. + filter_list = [column.split("__")[1:] for column in filter_list] + filter_list = ["__".join(item) for item in filter_list] + + add_list = [ + f"{self.param}__{suffix}" + for suffix in self.suffix_list + if f"{self.param}__{suffix}" in filter_list + ] + + if not self._distribution_check( + column_list, add_list, gdf, scenario=scenario + ): + # parameters niet voor dit scenario of niet correct ingevoerd. + continue + + scenario_add_list = [f"{scenario}__{item}" for item in add_list] + gdf_to_add = gdf[["geometry"] + scenario_add_list] + + for a, b in zip(scenario_add_list, add_list): + gdf_to_add = gdf_to_add.rename(columns={a: b}) + + gdf_to_add.to_file( + self.app_settings.geopackage_filepath, + layer=f"geom__{self.param}__{scenario}", + driver="GPKG", + ) + + return True + + def _add_separate_with_suffix(self, gdf: gpd.GeoDataFrame) -> bool: + """ + Voeg de lagen toe los van elkaar per scenario. + + :param gdf: GeoDataFrame met de uitgelezen data. + :return: bool voor valid_input in while-loop + """ + column_list = gdf.columns.to_list() + + if not self._scenario_check(column_list): + return False + + # 'None' als er geen scenario zijn opgegeven in de kolom + for scenario in self.scenarios + [None]: + if scenario is None: + mask = gdf["ondergrondscenario"].isna() + else: + mask = gdf["ondergrondscenario"] == scenario + + if not mask.any(): + continue + + add_list = [ + f"{self.param}__{suffix}" + for suffix in self.suffix_list + if f"{self.param}__{suffix}" in column_list + ] + + if not self._distribution_check(column_list, add_list, gdf): + return False + + gdf_to_add = gdf.loc[mask, ["geometry"] + add_list] + + if scenario is None: + gdf_to_add.to_file( + self.app_settings.geopackage_filepath, + layer=f"geom__{self.param}", + driver="GPKG", + ) + + else: + gdf_to_add.to_file( + self.app_settings.geopackage_filepath, + layer=f"geom__{self.param}__{scenario}", + driver="GPKG", + ) + + return True diff --git a/geoprob_pipe/cmd_app/spatial_layers/binnenteenlijn.py b/geoprob_pipe/cmd_app/spatial_layers/binnenteenlijn.py deleted file mode 100644 index 8e43a358..00000000 --- a/geoprob_pipe/cmd_app/spatial_layers/binnenteenlijn.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -TODO Later Should Klein: Controleer of de binnenteenlijn ook echt aan de binnenzijde is. -TODO Later Should Klein: Niet voor elk geohydrologisch model is de binnen/buiten/intredelijn benodigd. - Maak het toevoegen van deze lijnen afhankelijk van de model keuze. -""" - -from __future__ import annotations -from InquirerPy import inquirer -from typing import TYPE_CHECKING, Optional -import os -from shapely import LineString, MultiLineString -from geopandas import GeoDataFrame, read_file -import fiona -import warnings -from geoprob_pipe.utils.validation_messages import BColors -if TYPE_CHECKING: - from geoprob_pipe.cmd_app.cmd import ApplicationSettings - - -def added_binnenteenlijn(app_settings: ApplicationSettings) -> bool: - layers = fiona.listlayers(app_settings.geopackage_filepath) - - if "binnenteenlijn" in layers: - print(BColors.OKBLUE, f"✔ Binnenteenlijn al toegevoegd.", BColors.ENDC) - return True - - request_binnenteenlijn_filepath(app_settings=app_settings) - return True - - -def request_binnenteenlijn_filepath(app_settings: ApplicationSettings): - - # Request filepath - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: - filepath: str = inquirer.text( - message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " - "binnenteen lijnen zitten.", - ).execute() - - filepath = filepath.replace('"', '') - - if not (filepath.endswith(".gpkg") or filepath.endswith(".shp") or filepath.endswith(".gdb")): - print(BColors.WARNING, f"Het bestand moet of een geopackage, shapefile of geodatabase zijn. Jouw invoer " - f"eindigt op de extensie .{filepath.split(sep='.')[-1]}.", BColors.ENDC) - continue - if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) - continue - - filepath_is_valid = True - - # Import data - if filepath.endswith(".shp"): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") - gdf: GeoDataFrame = read_file(filepath) - elif filepath.endswith(".gpkg"): - gdf: GeoDataFrame = import_from_geopackage(filepath=filepath) - elif filepath.endswith(".gdb"): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") - gdf: GeoDataFrame = import_from_geodatabase(filepath=filepath) - else: - raise NotImplementedError( - f"Applicatie vroegtijdig afgesloten: Een {filepath.split(sep='.')[-1]}-bestand is niet geïmplementeerd.") - - # Confirm all are points - all_geometries_are_points = gdf.geometry.apply( - lambda geom: isinstance(geom, LineString) or isinstance(geom, MultiLineString)).all() - if not all_geometries_are_points: - print(BColors.WARNING, f"Het geïmporteerde bestand bestaat niet (volledig) uit lijnen, maar ook uit " - f"andere typen geometrie. Enkel lijnen zijn toegestaan.", BColors.ENDC) - request_binnenteenlijn_filepath(app_settings=app_settings) - - # Add binnenteenlijn - gdf_to_add = gdf[["geometry"]] - gdf_to_add.to_file(app_settings.geopackage_filepath, layer="binnenteenlijn", driver="GPKG") - print(BColors.OKBLUE, f"✅ Binnenteenlijn toegevoegd.", BColors.ENDC) - - -def import_from_geodatabase(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de layer waarin de binnenteenlijn staat. Type 'listlayers' om " - "een overzicht te krijgen van de geodatabase-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(f"{BColors.OKBLUE}De volgende layers zijn beschikbaar in de geodatabase: {layers_str}{BColors.ENDC}") - continue - elif layer_name not in layer_names: - print(f"{BColors.OKBLUE} De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geodatabase: {layers_str}{BColors.ENDC}") - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf - - -def import_from_geopackage(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de laag met de binnenteen lijnen. Type 'listlayers' om " - "een overzicht te krijgen van de geopackage-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geopackage: {layers_str}", BColors.ENDC) - continue - elif layer_name not in layer_names: - print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geopackage: {layers_str}", BColors.ENDC) - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf diff --git a/geoprob_pipe/cmd_app/spatial_layers/buitenteenlijn.py b/geoprob_pipe/cmd_app/spatial_layers/buitenteenlijn.py deleted file mode 100644 index 3aeaedc0..00000000 --- a/geoprob_pipe/cmd_app/spatial_layers/buitenteenlijn.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -TODO Later Should Klein: Controleer of de buitenteenlijn ook echt aan de buitenzijde is. - -""" - -from __future__ import annotations -from InquirerPy import inquirer -from typing import TYPE_CHECKING, Optional -import os -import warnings -from shapely import LineString, MultiLineString -from geopandas import GeoDataFrame, read_file -import fiona -from geoprob_pipe.utils.validation_messages import BColors -if TYPE_CHECKING: - from geoprob_pipe.cmd_app.cmd import ApplicationSettings - - -def added_buitenteenlijn(app_settings: ApplicationSettings) -> bool: - layers = fiona.listlayers(app_settings.geopackage_filepath) - - if "buitenteenlijn" in layers: - print(BColors.OKBLUE, f"✔ Buitenteenlijn al toegevoegd.", BColors.ENDC) - return True - - request_buitenteenlijn_filepath(app_settings=app_settings) - return True - - -def request_buitenteenlijn_filepath(app_settings: ApplicationSettings): - - # Request filepath - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: - filepath: str = inquirer.text( - message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " - "buitenteen lijnen zitten.", - ).execute() - - filepath = filepath.replace('"', '') - - if not (filepath.endswith(".gpkg") or filepath.endswith(".shp") or filepath.endswith(".gdb")): - print(BColors.WARNING, f"Het bestand moet of een geopackage, shapefile of geodatabase zijn. Jouw invoer " - f"eindigt op de extensie .{filepath.split(sep='.')[-1]}.", BColors.ENDC) - continue - if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) - continue - - filepath_is_valid = True - - # Import data - if filepath.endswith(".shp"): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") - gdf: GeoDataFrame = read_file(filepath) - elif filepath.endswith(".gpkg"): - gdf: GeoDataFrame = import_from_geopackage(filepath=filepath) - elif filepath.endswith(".gdb"): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") - gdf: GeoDataFrame = import_from_geodatabase(filepath=filepath) - else: - raise NotImplementedError( - f"Applicatie vroegtijdig afgesloten: Een {filepath.split(sep='.')[-1]}-bestand is niet geïmplementeerd.") - - # Confirm all are points - all_geometries_are_points = gdf.geometry.apply( - lambda geom: isinstance(geom, LineString) or isinstance(geom, MultiLineString)).all() - if not all_geometries_are_points: - print(BColors.WARNING, f"Het geïmporteerde bestand bestaat niet (volledig) uit lijnen, maar ook uit " - f"andere typen geometrie. Enkel lijnen zijn toegestaan.", BColors.ENDC) - request_buitenteenlijn_filepath(app_settings=app_settings) - - # Add buitenteenlijn - gdf_to_add = gdf[["geometry"]] - gdf_to_add.to_file(app_settings.geopackage_filepath, layer="buitenteenlijn", driver="GPKG") - print(BColors.OKBLUE, f"✅ Buitenteenlijn toegevoegd.", BColors.ENDC) - - -def import_from_geodatabase(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de layer waarin de buitenteenlijn staat. Type 'listlayers' om " - "een overzicht te krijgen van de geodatabase-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geodatabase: {layers_str}", BColors.ENDC) - continue - elif layer_name not in layer_names: - print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geodatabase: {layers_str}", BColors.ENDC) - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf - - -def import_from_geopackage(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de laag met de buitenteen lijnen. Type 'listlayers' om " - "een overzicht te krijgen van de geopackage-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geopackage: {layers_str}", BColors.ENDC) - continue - elif layer_name not in layer_names: - print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geopackage: {layers_str}", BColors.ENDC) - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf diff --git a/geoprob_pipe/cmd_app/spatial_layers/check_batch_input.py b/geoprob_pipe/cmd_app/spatial_layers/check_batch_input.py new file mode 100644 index 00000000..71eeb430 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/check_batch_input.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from geoprob_pipe.cmd_app.utils.batch_input import batch_inquiry, read_metadata + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + +def check_batch_input(app_settings: ApplicationSettings): + if not read_metadata(app_settings=app_settings): + return True + batch_inquiry(app_settings=app_settings) + return True \ No newline at end of file diff --git a/geoprob_pipe/cmd_app/spatial_layers/hrd/hrd.py b/geoprob_pipe/cmd_app/spatial_layers/hrd/hrd.py index 22f9939f..c32f0562 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/hrd/hrd.py +++ b/geoprob_pipe/cmd_app/spatial_layers/hrd/hrd.py @@ -35,7 +35,7 @@ def _hrd_data_requested(app_settings: ApplicationSettings) -> bool: if choice == "Nee": keuze = False cursor.execute( - f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') VALUES ('extract_hrd_data', {keuze});") + f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES ('extract_hrd_data', {keuze});") conn.commit() cursor.close() return keuze @@ -45,7 +45,7 @@ def _hrd_data_imported(app_settings: ApplicationSettings) -> bool: # Check if locations shape already added layers = fiona.listlayers(app_settings.geopackage_filepath) - if "hrd_locaties" not in layers: + if "geom__hrd_locaties" not in layers: return False # Check overschrijdingsfrequentielijnen already added @@ -54,9 +54,8 @@ def _hrd_data_imported(app_settings: ApplicationSettings) -> bool: cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") tables_names = [row[0] for row in cursor.fetchall()] conn.close() - if "fragility_values_invoer_hrd" not in tables_names: + if "data__fragility_values_invoer_hrd" not in tables_names: return False - return True diff --git a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py index 59f5450e..109716f1 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py +++ b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py @@ -40,9 +40,7 @@ def _folder_contains_hrd_db(hrd_dir: str) -> bool: def _ask_path_to_hrd_dir() -> str: - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: + while True: filepath: str = inquirer.text( message="Specificeer het volledige pad naar de bestandsmap met de Hydra-NL database. " "Dat zijn de hlcd, config en het database .sqlite-bestand zelf.", @@ -51,16 +49,16 @@ def _ask_path_to_hrd_dir() -> str: filepath = filepath.replace('"', '') if not os.path.isdir(filepath): - print(BColors.WARNING, f"Het bestandspad moet een directory zijn. Probeer het opnieuw.", BColors.ENDC) + print(BColors.WARNING, "Het bestandspad moet een directory zijn. Probeer het opnieuw.", BColors.ENDC) continue if not _folder_contains_hrd_db(hrd_dir=filepath): - print(BColors.WARNING, f"De opgegeven bestandsmap bevat geen of niet alle bestanden van de Hydra-NL " - f"database. Zorg er voor dat het hlcd-, config.sqlite- en het .sqlite " - f"database-bestand allen in de map staan. ", BColors.ENDC) + print(BColors.WARNING, "De opgegeven bestandsmap bevat geen of niet alle bestanden van de Hydra-NL " + "database. Zorg er voor dat het hlcd-, config.sqlite- en het .sqlite " + "database-bestand allen in de map staan. ", BColors.ENDC) continue - filepath_is_valid = True + break return filepath @@ -89,7 +87,7 @@ def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: s }) gdf = GeoDataFrame(hrd_location_rows, columns=['location_name', 'geometry'], crs='EPSG:28992') - gdf.to_file(Path(app_settings.geopackage_filepath), layer="hrd_locaties", driver="GPKG") + gdf.to_file(Path(app_settings.geopackage_filepath), layer="geom__hrd_locaties", driver="GPKG") print(BColors.OKBLUE, f"✅ HRD-locatie punten toegevoegd aan GeoProb-Pipe GeoPackage.", BColors.ENDC) @@ -98,7 +96,7 @@ def _add_hrd_overschrijdingsfrequentielijnen(hrd_dir: str, app_settings: Applica f"GeoPackage.{BColors.ENDC}") hrd = pydra.HRDatabase(hrd_file_path(hrd_dir=hrd_dir)) - gdf_locations: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="hrd_locaties") + gdf_locations: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__hrd_locaties") location_names = gdf_locations['location_name'].unique().tolist() fl = pydra.ExceedanceFrequencyLine("h") dfs: List[DataFrame] = [] @@ -134,7 +132,7 @@ def _add_hrd_overschrijdingsfrequentielijnen(hrd_dir: str, app_settings: Applica if dfs.__len__() > 0: df = concat(dfs, ignore_index=True) conn = sqlite3.connect(app_settings.geopackage_filepath) - df.to_sql("fragility_values_invoer_hrd", conn, if_exists="replace", index=False) + df.to_sql("data__fragility_values_invoer_hrd", conn, if_exists="replace", index=False) conn.close() print(BColors.OKBLUE, f"✅ HRD-overschrijdingsfrequentielijnen toegevoegd aan GeoProb-Pipe " @@ -202,14 +200,14 @@ def _add_traject_parameters(app_settings: ApplicationSettings, hrd_dir: str): conn = sqlite3.connect(file_path) cursor = conn.cursor() cursor.execute( - f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') VALUES ('traject_id', '{traject_id}');") + f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES ('traject_id', '{traject_id}');") cursor.execute( - f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') " + f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) " f"VALUES ('signaleringswaarde', {signaleringswaarde});") cursor.execute( - f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') VALUES ('ondergrens', {ondergrens});") - cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') VALUES ('w', {w});") - cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') " + f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES ('ondergrens', {ondergrens});") + cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES ('w', {w});") + cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) " f"VALUES ('is_bovenrivierengebied', {is_bovenrivierengebied});") conn.commit() cursor.close() diff --git a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_other_geopackage.py b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_other_geopackage.py index 2b704ab0..eafdf87f 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_other_geopackage.py +++ b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_other_geopackage.py @@ -14,9 +14,7 @@ def _ask_for_other_geopackage_file_path() -> str: - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: + while True: filepath: str = inquirer.text( message="Specificeer het volledige bestandspad naar het .geoprob_pipe.gpkg-bestand.", ).execute() @@ -27,28 +25,29 @@ def _ask_for_other_geopackage_file_path() -> str: print(BColors.WARNING, f"Het bestand moet een .geoprob_pipe.gpkg-bestand zijn. Jouw invoer " f"{os.path.basename(filepath)} eindigt niet op deze extensie.", BColors.ENDC) continue + if not os.path.exists(filepath): print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) continue - filepath_is_valid = True + break return filepath def _add_hrd_locations_to_database(other_gpkg_path: str, app_settings: ApplicationSettings): - gdf_locations: GeoDataFrame = read_file(other_gpkg_path, layer="hrd_locaties") - gdf_locations.to_file(Path(app_settings.geopackage_filepath), layer="hrd_locaties", driver="GPKG") + gdf_locations: GeoDataFrame = read_file(other_gpkg_path, layer="geom__hrd_locaties") + gdf_locations.to_file(Path(app_settings.geopackage_filepath), layer="geom__hrd_locaties", driver="GPKG") print(BColors.OKBLUE, f"✅ HRD-locatie punten toegevoegd aan GeoProb-Pipe GeoPackage.", BColors.ENDC) def _add_hrd_overschrijdingsfrequentielijnen(other_gpkg_path: str, app_settings: ApplicationSettings): conn = sqlite3.connect(other_gpkg_path) - df: DataFrame = read_sql_query("SELECT * FROM fragility_values_invoer_hrd;", conn) + df: DataFrame = read_sql_query("SELECT * FROM data__fragility_values_invoer_hrd;", conn) conn.close() conn = sqlite3.connect(app_settings.geopackage_filepath) - df.to_sql("fragility_values_invoer_hrd", conn, if_exists="replace", index=False) + df.to_sql("data__fragility_values_invoer_hrd", conn, if_exists="replace", index=False) conn.close() print(BColors.OKBLUE, f"✅ HRD-overschrijdingsfrequentielijnen toegevoegd aan GeoProb-Pipe " @@ -60,23 +59,23 @@ def _add_traject_parameters(other_gpkg_path: str, app_settings: ApplicationSetti df: DataFrame = read_sql_query("SELECT * FROM geoprob_pipe_metadata;", conn) conn.close() - traject_id = df[df["metadata_type"] == "traject_id"]["values"].values[0] - signaleringswaarde = int(df[df["metadata_type"] == "signaleringswaarde"]["values"].values[0]) - ondergrens = int(df[df["metadata_type"] == "ondergrens"]["values"].values[0]) - w = float(df[df["metadata_type"] == "w"]["values"].values[0]) - is_bovenrivierengebied = bool(int(df[df["metadata_type"] == "is_bovenrivierengebied"]["values"].values[0])) + traject_id = df[df["metadata_type"] == "traject_id"]["metadata_value"].values[0] + signaleringswaarde = int(df[df["metadata_type"] == "signaleringswaarde"]["metadata_value"].values[0]) + ondergrens = int(df[df["metadata_type"] == "ondergrens"]["metadata_value"].values[0]) + w = float(df[df["metadata_type"] == "w"]["metadata_value"].values[0]) + is_bovenrivierengebied = bool(int(df[df["metadata_type"] == "is_bovenrivierengebied"]["metadata_value"].values[0])) conn = sqlite3.connect(app_settings.geopackage_filepath) cursor = conn.cursor() cursor.execute( - f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') VALUES ('traject_id', '{traject_id}');") + f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES ('traject_id', '{traject_id}');") cursor.execute( - f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') " + f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) " f"VALUES ('signaleringswaarde', {signaleringswaarde});") cursor.execute( - f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') VALUES ('ondergrens', {ondergrens});") - cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') VALUES ('w', {w});") - cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, 'values') " + f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES ('ondergrens', {ondergrens});") + cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES ('w', {w});") + cursor.execute(f"INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) " f"VALUES ('is_bovenrivierengebied', {is_bovenrivierengebied});") conn.commit() cursor.close() diff --git a/geoprob_pipe/cmd_app/spatial_layers/intredelijn.py b/geoprob_pipe/cmd_app/spatial_layers/intredelijn.py deleted file mode 100644 index 94269593..00000000 --- a/geoprob_pipe/cmd_app/spatial_layers/intredelijn.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -TODO Later Should Klein: Controleer of de intredelijn ook echt aan de binnenzijde is. - -""" - -from __future__ import annotations -from InquirerPy import inquirer -from typing import TYPE_CHECKING, Optional -import warnings -import os -from shapely import LineString, MultiLineString -from geopandas import GeoDataFrame, read_file -import fiona -from geoprob_pipe.utils.validation_messages import BColors -if TYPE_CHECKING: - from geoprob_pipe.cmd_app.cmd import ApplicationSettings - - -def added_intredelijn(app_settings: ApplicationSettings) -> bool: - layers = fiona.listlayers(app_settings.geopackage_filepath) - - if "intredelijn" in layers: - print(BColors.OKBLUE, f"✔ Intredelijn al toegevoegd.", BColors.ENDC) - return True - - request_intredelijn_filepath(app_settings=app_settings) - return True - - -def request_intredelijn_filepath(app_settings: ApplicationSettings): - - # Request filepath - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: - filepath: str = inquirer.text( - message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " - "intrede lijnen zitten.", - ).execute() - - filepath = filepath.replace('"', '') - - if not (filepath.endswith(".gpkg") or filepath.endswith(".shp") or filepath.endswith(".gdb")): - print(BColors.WARNING, f"Het bestand moet of een geopackage, shapefile of geodatabase zijn. Jouw invoer " - f"eindigt op de extensie .{filepath.split(sep='.')[-1]}.", BColors.ENDC) - continue - if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) - continue - - filepath_is_valid = True - - # Import data - if filepath.endswith(".shp"): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") - gdf: GeoDataFrame = read_file(filepath) - elif filepath.endswith(".gpkg"): - gdf: GeoDataFrame = import_from_geopackage(filepath=filepath) - elif filepath.endswith(".gdb"): - gdf: GeoDataFrame = import_from_geodatabase(filepath=filepath) - else: - raise NotImplementedError( - f"Applicatie vroegtijdig afgesloten: Een {filepath.split(sep='.')[-1]}-bestand is niet geïmplementeerd.") - - # Confirm all are points - all_geometries_are_points = gdf.geometry.apply( - lambda geom: isinstance(geom, LineString) or isinstance(geom, MultiLineString)).all() - if not all_geometries_are_points: - print(BColors.WARNING, f"Het geïmporteerde bestand bestaat niet (volledig) uit lijnen, maar ook uit " - f"andere typen geometrie. Enkel lijnen zijn toegestaan.", BColors.ENDC) - request_intredelijn_filepath(app_settings=app_settings) - - # Add intredelijn - gdf_to_add = gdf[["geometry"]] - gdf_to_add.to_file(app_settings.geopackage_filepath, layer="intredelijn", driver="GPKG") - print(BColors.OKBLUE, f"✅ Intredelijn toegevoegd.", BColors.ENDC) - - -def import_from_geodatabase(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de layer waarin de intredelijn staat. Type 'listlayers' om " - "een overzicht te krijgen van de geodatabase-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geodatabase: {layers_str}", BColors.ENDC) - continue - elif layer_name not in layer_names: - print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geodatabase: {layers_str}", BColors.ENDC) - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf - - -def import_from_geopackage(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de laag met de intrede lijnen. Type 'listlayers' om " - "een overzicht te krijgen van de geopackage-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geopackage: {layers_str}", BColors.ENDC) - continue - elif layer_name not in layer_names: - print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geopackage: {layers_str}", BColors.ENDC) - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf diff --git a/geoprob_pipe/cmd_app/spatial_layers/polderpeil.py b/geoprob_pipe/cmd_app/spatial_layers/polderpeil.py deleted file mode 100644 index ad73a421..00000000 --- a/geoprob_pipe/cmd_app/spatial_layers/polderpeil.py +++ /dev/null @@ -1,149 +0,0 @@ -from __future__ import annotations -from InquirerPy import inquirer -from typing import TYPE_CHECKING, Optional -import os -from shapely import Polygon, MultiPolygon -from geopandas import GeoDataFrame, read_file -import fiona -from geoprob_pipe.utils.validation_messages import BColors -import warnings -if TYPE_CHECKING: - from geoprob_pipe.cmd_app.cmd import ApplicationSettings - - -def added_polderpeil(app_settings: ApplicationSettings) -> bool: - layers = fiona.listlayers(app_settings.geopackage_filepath) - - if "polderpeil" in layers: - print(BColors.OKBLUE, f"✔ Polderpeil al toegevoegd.", BColors.ENDC) - return True - - request_polderpeil_filepath(app_settings=app_settings) - return True - - -def request_polderpeil_filepath(app_settings: ApplicationSettings): - - # Request filepath - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: - filepath: str = inquirer.text( - message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " - "polderpeilen zitten.", - ).execute() - - filepath = filepath.replace('"', '') - - if not (filepath.endswith(".gpkg") or filepath.endswith(".shp") or filepath.endswith(".gdb")): - print(BColors.WARNING, f"Het bestand moet of een geopackage, shapefile of geodatabase zijn. Jouw invoer " - f"eindigt op de extensie .{filepath.split(sep='.')[-1]}.", BColors.ENDC) - continue - if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) - continue - - filepath_is_valid = True - - # Import data - if filepath.endswith(".shp"): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") - gdf: GeoDataFrame = read_file(filepath) - elif filepath.endswith(".gpkg"): - gdf: GeoDataFrame = import_from_geopackage(filepath=filepath) - elif filepath.endswith(".gdb"): - gdf: GeoDataFrame = import_from_geodatabase(filepath=filepath) - else: - raise NotImplementedError( - f"Applicatie vroegtijdig afgesloten: Een {filepath.split(sep='.')[-1]}-bestand is niet geïmplementeerd.") - - # Confirm all are points - all_geometries_are_points = gdf.geometry.apply( - lambda geom: isinstance(geom, Polygon) or isinstance(geom, MultiPolygon)).all() - if not all_geometries_are_points: - print(BColors.WARNING, f"Het geïmporteerde bestand bestaat niet (volledig) uit vlakken/polygonen, maar ook uit " - f"andere typen geometrie. Enkel vlakken/polygonen zijn toegestaan.", BColors.ENDC) - request_polderpeil_filepath(app_settings=app_settings) - - # Continue questionnaire - specify_column_with_polderpeil_niveau(app_settings, gdf=gdf) - - -def import_from_geodatabase(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de layer waarin met het polderpeil. Type 'listlayers' om " - "een overzicht te krijgen van de geodatabase-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geodatabase: {layers_str}", BColors.ENDC) - continue - elif layer_name not in layer_names: - print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geodatabase: {layers_str}", BColors.ENDC) - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf - - -def import_from_geopackage(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de laag met de polderpeilen. Type 'listlayers' om " - "een overzicht te krijgen van de geopackage-layers. ", - ).execute() - - layer_names = fiona.listlayers(filepath) - layer_names.sort() - layers_str = ", ".join(layer_names) - if layer_name == "listlayers": - print(BColors.OKBLUE, f"De volgende layers zijn beschikbaar in de geopackage: {layers_str}", BColors.ENDC) - continue - elif layer_name not in layer_names: - print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " - f"de geopackage: {layers_str}", BColors.ENDC) - continue - - layer_name_is_valid = True - - gdf: GeoDataFrame = read_file(filepath, layer=layer_name) - return gdf - - -def specify_column_with_polderpeil_niveau(app_settings: ApplicationSettings, gdf: GeoDataFrame): - column_name: Optional[str] = None - column_name_is_valid = False - while column_name_is_valid is False: - column_name: str = inquirer.text( - message="Specificeer de kolom waarin het polderpeil staat. Type 'listcolumns' om " - "een overzicht te krijgen van de kolommen.", - ).execute() - - column_names = gdf.columns - columns_str = ", ".join(column_names) - if column_name == "listcolumns": - print(BColors.OKBLUE, - f"De volgende kolommen zijn beschikbaar in de spatial layer: {columns_str}", BColors.ENDC) - continue - elif column_name not in column_names: - print(BColors.OKBLUE, f"De kolom naam '{column_name}' bestaat niet. De volgende kolommen zijn beschikbaar " - f"in de spatial layer: {columns_str}", BColors.ENDC) - continue - column_name_is_valid = True - - gdf_to_add = gdf[["geometry", column_name]] - gdf_to_add = gdf_to_add.rename(columns={column_name: "polderpeil"}) - gdf_to_add.to_file(app_settings.geopackage_filepath, layer="polderpeil", driver="GPKG") - print(BColors.OKBLUE, f"✅ Polderpeilen toegevoegd.", BColors.ENDC) diff --git a/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/alg_walking_circles.py b/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/alg_walking_circles.py index 7eb99ca4..269959f9 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/alg_walking_circles.py +++ b/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/alg_walking_circles.py @@ -1,5 +1,5 @@ from __future__ import annotations -from geoprob_pipe.cmd_app.spatial_layers.ahn import added_ahn, request_ahn +from geoprob_pipe.cmd_app.spatial_layers.added_ahn import added_ahn, request_ahn from typing import TYPE_CHECKING, Optional, Tuple, List from geoprob_pipe.cmd_app.utils.spatial import load_dijktraject_linestring, load_hydra_nl_as_multipoint from shapely import LineString, MultiPoint, Polygon, MultiPolygon, Point, unary_union @@ -19,26 +19,27 @@ def request_buffer_distance() -> int: int_distance = 50 - distance_is_valid = False - while distance_is_valid is False: + while True: str_distance: str = inquirer.text( message="Specificeer de afstand (in meters) waarbinnen gezocht moet worden aan binnendijkse zijde.\n" "Zorg er voor dat deze afstand volledig binnen het toegevoegde raster valt. ", ).execute() # Validate - try: int_distance = int(str_distance) + try: + int_distance = int(str_distance) except ValueError: print(f"{BColors.WARNING}Het lukte niet om de invoer om te zetten naar een integer (geheel getal). Weet je " f"zeker dat de invoer correct is? Probeer het opnieuw.{BColors.ENDC}") continue + if int_distance < 50: print(f"{BColors.WARNING}De opgegeven afstand moet ten minste 50 meter zijn. " f"Vul opnieuw een waarde in. {BColors.ENDC}") continue # Valid - distance_is_valid = True + break return int_distance @@ -195,5 +196,5 @@ def algorithm_walking_circles(app_settings: ApplicationSettings): gdf_to_add = gdf_proposed_uittredepunten[["geometry", "min_val"]] gdf_to_add = gdf_to_add.rename(columns={"min_val": "mv_exit"}) - gdf_to_add.to_file(app_settings.geopackage_filepath, layer="uittredepunten", driver="GPKG") + gdf_to_add.to_file(app_settings.geopackage_filepath, layer="geom__uittredepunten", driver="GPKG") print(BColors.OKBLUE, f"✅ Uittredepunten toegevoegd.", BColors.ENDC) diff --git a/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py b/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py index 2d7a2733..d1dc7ac5 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py +++ b/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py @@ -1,6 +1,6 @@ from __future__ import annotations from InquirerPy import inquirer -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import os import sys from geoprob_pipe.cmd_app.spatial_layers.uittredepunten.alg_walking_circles import algorithm_walking_circles @@ -16,11 +16,11 @@ def added_uittredepunten(app_settings: ApplicationSettings) -> bool: layers = fiona.listlayers(app_settings.geopackage_filepath) - if "uittredepunten" not in layers: + if "geom__uittredepunten" not in layers: define_method_of_adding_uittredepunten(app_settings) return True - gdf: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="uittredepunten") + gdf: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__uittredepunten") if gdf.__len__() == 0: define_method_of_adding_uittredepunten(app_settings) return True @@ -53,7 +53,7 @@ def define_method_of_adding_uittredepunten(app_settings: ApplicationSettings): # elif choice == choices_list[2]: # generate_uittredepunten_suggestions(app_settings=app_settings) elif choice == choices_list[1]: - sys.exit(f"Applicatie is afgesloten.") + sys.exit("Applicatie is afgesloten.") else: raise ValueError @@ -72,14 +72,19 @@ def import_uittredepunten_gis_file(app_settings: ApplicationSettings): request_uittredepunten_filepath(app_settings=app_settings) -def import_from_geodatabase(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de layer waarin met de uittredepunten. Type 'listlayers' om " - "een overzicht te krijgen van de geodatabase-layers. ", - ).execute() +def import_from_geodatabase(app_settings: ApplicationSettings, filepath: str) -> GeoDataFrame: + layer_name: str = "" + skip_batch = False # Als batch input faalt ga over op handmatig + while True: + if app_settings.batch_input and not skip_batch: + layer_name = app_settings.input_config.get( + "uittredepunten", "database_layer" + ) + else: + layer_name: str = inquirer.text( + message="Specificeer de layer waarin met de uittredepunten. Type 'listlayers' om " + "een overzicht te krijgen van de geodatabase-layers. ", + ).execute() layer_names = fiona.listlayers(filepath) layer_names.sort() @@ -90,22 +95,29 @@ def import_from_geodatabase(filepath: str) -> GeoDataFrame: elif layer_name not in layer_names: print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " f"de geodatabase: {layers_str}", BColors.ENDC) + skip_batch = True continue - layer_name_is_valid = True + break gdf: GeoDataFrame = read_file(filepath, layer=layer_name) return gdf -def import_from_geopackage(filepath: str) -> GeoDataFrame: - layer_name: Optional[str] = None - layer_name_is_valid = False - while layer_name_is_valid is False: - layer_name: str = inquirer.text( - message="Specificeer de layer waarin met de uittredepunten. Type 'listlayers' om " - "een overzicht te krijgen van de geopackage-layers. ", - ).execute() +def import_from_geopackage(app_settings: ApplicationSettings, filepath: str) -> GeoDataFrame: + layer_name: str = "" + skip_batch = False # Als batch input faalt ga over op handmatig + + while True: + if app_settings.batch_input and not skip_batch: + layer_name = app_settings.input_config.get( + "uittredepunten", "database_layer" + ) + else: + layer_name: str = inquirer.text( + message="Specificeer de layer waarin met de uittredepunten. Type 'listlayers' om " + "een overzicht te krijgen van de geopackage-layers. ", + ).execute() layer_names = fiona.listlayers(filepath) layer_names.sort() @@ -116,9 +128,10 @@ def import_from_geopackage(filepath: str) -> GeoDataFrame: elif layer_name not in layer_names: print(BColors.OKBLUE, f"De laag name '{layer_name}' bestaat niet. De volgende layers zijn beschikbaar in " f"de geopackage: {layers_str}", BColors.ENDC) + skip_batch = True continue - layer_name_is_valid = True + break with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="Measured \\(M\\) geometry types are not supported.*") @@ -150,13 +163,19 @@ def generate_uittredepunten_suggestions(app_settings: ApplicationSettings): def request_uittredepunten_filepath(app_settings: ApplicationSettings): # Request filepath - filepath: Optional[str] = None - filepath_is_valid = False - while filepath_is_valid is False: - filepath: str = inquirer.text( - message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " - "uittredepunten zitten.", - ).execute() + filepath: str = "" + skip_batch = False # Als batch input faalt ga over op handmatig + + while True: + if app_settings.batch_input and not skip_batch: + filepath = app_settings.input_config.get( + "uittredepunten", "filepath" + ) + else: + filepath: str = inquirer.text( + message="Specificeer het volledige bestandspad naar de geopackage/shapefile/geodatabase waarin de " + "uittredepunten zitten.", + ).execute() filepath = filepath.replace('"', '') @@ -165,18 +184,21 @@ def request_uittredepunten_filepath(app_settings: ApplicationSettings): f"eindigt op de extensie .{filepath.split(sep='.')[-1]}.", BColors.ENDC) continue if not os.path.exists(filepath): - print(BColors.WARNING, f"Het opgegeven bestandspad bestaat niet.", BColors.ENDC) + print(BColors.WARNING, "Het opgegeven bestandspad bestaat niet.", BColors.ENDC) + skip_batch = True continue - filepath_is_valid = True + break # Import data if filepath.endswith(".shp"): gdf: GeoDataFrame = read_file(filepath) elif filepath.endswith(".gpkg"): - gdf: GeoDataFrame = import_from_geopackage(filepath=filepath) + gdf: GeoDataFrame = import_from_geopackage(app_settings=app_settings, + filepath=filepath) elif filepath.endswith(".gdb"): - gdf: GeoDataFrame = import_from_geodatabase(filepath=filepath) + gdf: GeoDataFrame = import_from_geodatabase(app_settings=app_settings, + filepath=filepath) else: raise NotImplementedError( f"Applicatie vroegtijdig afgesloten: Een {filepath.split(sep='.')[-1]}-bestand is nog niet " @@ -194,14 +216,19 @@ def request_uittredepunten_filepath(app_settings: ApplicationSettings): def specify_column_with_maaiveld_niveau(app_settings: ApplicationSettings, gdf: GeoDataFrame): - column_name: Optional[str] = None - column_name_is_valid = False - while column_name_is_valid is False: - column_name: str = inquirer.text( - message="Specificeer de kolom waarin het maaiveld niveau staat. Type 'listcolumns' om " - "een overzicht te krijgen van de kolommen. Type 'n.a.' als het maaiveld niveau niet gekoppeld is " - "aan de punten, dan zal de applicatie in een volgende stap deze voor je downloaden.", - ).execute() + column_name: str = "" + skip_batch = False + while True: + if app_settings.batch_input and not skip_batch: + column_name = app_settings.input_config.get( + "uittredepunten", "mv_exit_kolom" + ) + else: + column_name: str = inquirer.text( + message="Specificeer de kolom waarin het maaiveld niveau staat. Type 'listcolumns' om " + "een overzicht te krijgen van de kolommen. Type 'n.a.' als het maaiveld niveau niet gekoppeld is " + "aan de punten, dan zal de applicatie in een volgende stap deze voor je downloaden.", + ).execute() column_names = gdf.columns columns_str = ", ".join(column_names) @@ -212,17 +239,18 @@ def specify_column_with_maaiveld_niveau(app_settings: ApplicationSettings, gdf: elif column_name == "n.a.": gdf_to_add = gdf[["geometry"]] gdf_to_add["mv_exit"] = -999.9 - gdf_to_add.to_file(app_settings.geopackage_filepath, layer="uittredepunten", driver="GPKG") + gdf_to_add.to_file(app_settings.geopackage_filepath, layer="geom__uittredepunten", driver="GPKG") print(BColors.OKBLUE, f"✅ Uittredepunten toegevoegd.", BColors.ENDC) return elif column_name not in column_names: print(BColors.OKBLUE, f"De kolom naam '{column_name}' bestaat niet. De volgende kolommen zijn beschikbaar " f"in de spatial layer: {columns_str}", BColors.ENDC) + skip_batch = True continue - column_name_is_valid = True + break gdf_to_add = gdf[["geometry", column_name]] # No uittredepunt id yet. This will be set when the metering is determined. gdf_to_add = gdf_to_add.rename(columns={column_name: "mv_exit"}) - gdf_to_add.to_file(app_settings.geopackage_filepath, layer="uittredepunten", driver="GPKG") + gdf_to_add.to_file(app_settings.geopackage_filepath, layer="geom__uittredepunten", driver="GPKG") print(BColors.OKBLUE, f"✅ Uittredepunten toegevoegd.", BColors.ENDC) diff --git a/geoprob_pipe/cmd_app/spatial_layers/valid_parameters.py b/geoprob_pipe/cmd_app/spatial_layers/valid_parameters.py new file mode 100644 index 00000000..c7fab146 --- /dev/null +++ b/geoprob_pipe/cmd_app/spatial_layers/valid_parameters.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +from geoprob_pipe.calculations.systems.model4a.initial_input import ( + INITIAL_INPUT as MODEL4A_INPUT, +) +from geoprob_pipe.calculations.systems.moria.initial_input import ( + INITIAL_INPUT as MORIA_INPUT, +) +from geoprob_pipe.calculations.systems.wbi.initial_input import ( + INITIAL_INPUT as WBI_INPUT, +) + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + +# Lijst van parameters die verwerkt kunnen worden: +# Polygon of raster alleen beschikbaar als iets per uittredepunt kan verschillen +# en afstand tot de keringlijn uitmaakt. +LIST_PARAMS: dict[str, dict] = { + "mv_exit": {"shape": ["raster"]}, + "polderpeil": {"shape": ["line", "polygon", "raster"]}, + "buitenwaterstand_gemiddeld": {"shape": ["line"]}, + "phi_exit_gemiddeld": {"shape": ["line", "polygon", "raster"]}, + "r_exit": {"shape": ["line", "polygon", "raster"]}, + "k_wvp": {"shape": ["line"]}, + "kD_wvp": {"shape": ["line"]}, + "modelfactor_h": {"shape": ["line"]}, + "modelfactor_ff": {"shape": ["line"]}, + "modelfactor_3d": {"shape": ["line"]}, + "modelfactor_ml": {"shape": ["line"]}, + "i_c_h": {"shape": ["line"]}, + "modelfactor_u": {"shape": ["line"]}, + "modelfactor_p": {"shape": ["line"]}, + "d70": {"shape": ["line"]}, + "c_voorland": {"shape": ["line"]}, + "c_achterland": {"shape": ["line"]}, + "a_vak": {"shape":["line"]}, + "delta_length": {"shape":["line"]}, +} + +def valid_parameter_list(app_settings: ApplicationSettings): + """ + Bepaal de lijst met parameters op basis van de ingevoerde lijst en de parameters + die in het gekozen model gebruikt worden. + + :param app_settings: Object met de instellingen van de app. + :return: Lijst met de parameters die ruimtelijk ingevoerd kunnen worden. + """ + conn = sqlite3.connect(app_settings.geopackage_filepath) + cur = conn.cursor() + cur.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type='geohydrologisch_model'" + ) + model = cur.fetchone()[0] + conn.close() + + input_list: list[str] = [param for param in LIST_PARAMS.keys()] + valid_list = [] + match model: + case "model4a": + model_list = [param["name"] for param in MODEL4A_INPUT] + valid_list = [param for param in input_list if param in model_list] + + case "wbi": + model_list = [param["name"] for param in WBI_INPUT] + valid_list = [param for param in input_list if param in model_list] + + case "moria": + model_list = [param["name"] for param in MORIA_INPUT] + valid_list = [param for param in input_list if param in model_list] + + return valid_list diff --git a/geoprob_pipe/cmd_app/utils/batch_input.py b/geoprob_pipe/cmd_app/utils/batch_input.py new file mode 100644 index 00000000..f67712c8 --- /dev/null +++ b/geoprob_pipe/cmd_app/utils/batch_input.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import configparser +import sqlite3 +import sys +from typing import TYPE_CHECKING + +from InquirerPy.prompts.list import ListPrompt + +if TYPE_CHECKING: + from geoprob_pipe.cmd_app.cmd import ApplicationSettings + + +def batch_inquiry(app_settings: ApplicationSettings): + """ + Keuzemenu voor het gebruik van ee batch input bestand of handmatige invoer. + + :param app_settings: _description_ + """ + choices_list = [ + "Nee, handmatig invoeren", + "Ja, invoeren via `batch_input.ini` bestand", + "Maak het `batch_input.ini` bestand aan", + ] + while True: + choice = ListPrompt( + message=( + "Wil je de gis-lagen handmatig invoeren of via een batch_input.ini bestand?" + ), + choices=choices_list, + default=choices_list[0], + ).execute() + + match choice: + case "Nee, handmatig invoeren": + update_batch_metadata(app_settings=app_settings, value=False) + return + case "Ja, invoeren via `batch_input.ini` bestand": + app_settings.batch_input = True + return + case "Maak het `batch_input.ini` bestand aan": + batch_config_writer(app_settings) + app_settings.batch_input = True + ready_inquiry() + return + + +def ready_inquiry(): + """ + Keuzemenu als wacht periode voor het invullen van batch_input.ini. + """ + choices_list = [ + "batch_input.ini is klaar voor uitlezen", + "applicatie afsluiten", + ] + while True: + choise = ListPrompt( + message="Is het batch_input.ini bestand klaar voor gebruik?", + choices=choices_list, + default=choices_list[0], + ).execute() + match choise: + case "batch_input.ini is klaar voor uitlezen": + return + case "applicatie afsluiten": + sys.exit("Applicatie is afgesloten.") + + +def batch_config_writer(app_settings: ApplicationSettings): + """ + Functie om het bestand op te zetten voor alle benodigde handelingen. + + :param app_settings: _description_ + """ + conn = sqlite3.connect(app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("ruimtelijke_parameters",), + ) + parameters: list[str] = cursor.fetchone()[0].split(", ") + conn.close() + + config = configparser.ConfigParser() + + config["vakindeling"] = { + "filepath": "", + "database_layer": "", + "vak_naam_kolom": "", + "vak_id_kolom": "", + } + + config["uittredepunten"] = { + "filepath": "", + "database_layer": "", + "mv_exit_kolom": "", + } + + config["binnenteenlijn"] = { + "filepath": "", + "database_layer": "", + } + + config["buitenteenlijn"] = { + "filepath": "", + "database_layer": "", + } + + config["intredelijn"] = { + "filepath": "", + "database_layer": "", + } + + if not parameters == [""]: + for parameter in parameters: + config[parameter] = {"filepath": "", "database_layer": ""} + + with open(f"{app_settings.workspace_dir}/batch_input.ini", "w") as f: + f.write(""" +# Dit is een bestand om een deel van de invoer van GeoProb-Pipe in +# een keer in te voeren. Hier moeten dezelfde paden, lagen en kolommen +# worden ingevoerd als met de handmatige invoer. + +# Geef bij 'filepath' het volledige bestandspad op naar het bestand met de invoer. +# Als het bestand een shapefile is mag je de volgende waarde leeg houden. +# Deze wordt alleen uitgelezen bij een database(.gdb of .gpkg) +# Geef bij 'database_layer' de naam op van de laag waarin de invoer staat. +# Soms moet er ook nog een kolom worden opgegeven: +# Geef bij '*_kolom' de kolom naam op waarin de opgevraagde parameter in staat. + +""") + config.write(f) + +def update_batch_metadata(app_settings: ApplicationSettings, value: bool): + conn = sqlite3.connect(app_settings.geopackage_filepath) + + sql_update = "UPDATE geoprob_pipe_metadata SET metadata_value = ? WHERE metadata_type = ?" + sql_insert = "INSERT INTO geoprob_pipe_metadata (metadata_type, metadata_value) VALUES (?, ?)" + + with conn: # transaction + cur = conn.execute(sql_update, (value, "batch_inquiry")) # type:ignore + if cur.rowcount == 0: + conn.execute(sql_insert, ("batch_inquiry", value)) # type:ignore + conn.execute(sql_update, (value, "batch_inquiry")) + + conn.commit() + conn.close() + +def read_metadata(app_settings: ApplicationSettings): + conn = sqlite3.connect(app_settings.geopackage_filepath) + cursor = conn.cursor() + cursor.execute( + "SELECT metadata_value FROM geoprob_pipe_metadata WHERE metadata_type = ?", + ("batch_inquiry",), + ) + row = cursor.fetchone() + conn.close() + if row is None: + return True + + return bool(int(row[0])) \ No newline at end of file diff --git a/geoprob_pipe/cmd_app/utils/misc.py b/geoprob_pipe/cmd_app/utils/misc.py index 63ba4716..b5035eb2 100644 --- a/geoprob_pipe/cmd_app/utils/misc.py +++ b/geoprob_pipe/cmd_app/utils/misc.py @@ -18,7 +18,7 @@ def get_geohydrological_model(app_settings: ApplicationSettings) -> str: conn = sqlite3.connect(app_settings.geopackage_filepath) cursor = conn.cursor() cursor.execute(""" - SELECT geoprob_pipe_metadata."values" + SELECT geoprob_pipe_metadata.metadata_value FROM geoprob_pipe_metadata WHERE metadata_type='geohydrologisch_model'; """) diff --git a/geoprob_pipe/cmd_app/utils/spatial.py b/geoprob_pipe/cmd_app/utils/spatial.py index 4d1f07a0..09e373c0 100644 --- a/geoprob_pipe/cmd_app/utils/spatial.py +++ b/geoprob_pipe/cmd_app/utils/spatial.py @@ -8,7 +8,7 @@ def load_dijktraject_linestring(app_settings: ApplicationSettings) -> LineString: - gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="dijktraject") + gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__dijktraject") gdf_dijktraject_geom = gdf_dijktraject.iloc[0].geometry if isinstance(gdf_dijktraject_geom, MultiLineString): assert gdf_dijktraject_geom.geoms.__len__() == 1 @@ -22,5 +22,5 @@ def load_dijktraject_linestring(app_settings: ApplicationSettings) -> LineString def load_hydra_nl_as_multipoint(app_settings: ApplicationSettings) -> MultiPoint: - gdf_hrd_locaties: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="hrd_locaties") + gdf_hrd_locaties: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__hrd_locaties") return MultiPoint(gdf_hrd_locaties.geometry.values) diff --git a/geoprob_pipe/input_data/input_data_object.py b/geoprob_pipe/input_data/input_data_object.py index 6b917071..f6558fcd 100644 --- a/geoprob_pipe/input_data/input_data_object.py +++ b/geoprob_pipe/input_data/input_data_object.py @@ -24,7 +24,7 @@ class Uittredepunten: def __init__(self, app_settings: ApplicationSettings): self.app_settings: ApplicationSettings = app_settings self.gdf: GeoDataFrame = read_file(app_settings.geopackage_filepath, - layer="uittredepunten") + layer="geom__uittredepunten") def uittredepunt(self, uittredepunt_id: int): row = self.gdf.loc[ @@ -57,7 +57,7 @@ def __init__(self, app_settings: ApplicationSettings): def _query_scenarios(self) -> DataFrame: conn = sqlite3.connect(self.app_settings.geopackage_filepath) - df_scenario_invoer = read_sql("SELECT * FROM scenario_invoer;", conn) + df_scenario_invoer = read_sql("SELECT * FROM data__scenario_invoer;", conn) df_scenario_invoer = df_scenario_invoer[["vak_id", "naam", "kans"]] conn.close() return df_scenario_invoer @@ -75,14 +75,14 @@ class HydraNLData: def __init__(self, app_settings: ApplicationSettings): self.app_settings: ApplicationSettings = app_settings self.gdf_locations: GeoDataFrame = read_file( - app_settings.geopackage_filepath, layer="hrd_locaties") + app_settings.geopackage_filepath, layer="geom__hrd_locaties") def hrd_fragility_values(self, ref: str) -> List[FragilityValue]: # Read from geopackage conn = sqlite3.connect(self.app_settings.geopackage_filepath) df_frag_line = read_sql( - sql=f"SELECT * FROM fragility_values_invoer_hrd WHERE fragility_values_ref = '{ref}' AND kans < 1.0;", + sql=f"SELECT * FROM data__fragility_values_invoer_hrd WHERE fragility_values_ref = '{ref}' AND kans < 1.0;", con=conn) conn.close() @@ -120,7 +120,7 @@ class Vakken: def __init__(self, app_settings: ApplicationSettings): self.app_settings: ApplicationSettings = app_settings self.gdf: GeoDataFrame = read_file(app_settings.geopackage_filepath, - layer="vakindeling") + layer="geom__vakindeling") class InputData: @@ -150,7 +150,7 @@ def geohydrologisch_model(self) -> str: conn = sqlite3.connect(self.app_settings.geopackage_filepath) cursor = conn.cursor() cursor.execute(""" - SELECT geoprob_pipe_metadata."values" + SELECT geoprob_pipe_metadata.metadata_value FROM geoprob_pipe_metadata WHERE metadata_type='geohydrologisch_model'; """) diff --git a/geoprob_pipe/input_data/traject_normering.py b/geoprob_pipe/input_data/traject_normering.py index 07ef8cdb..d001122f 100644 --- a/geoprob_pipe/input_data/traject_normering.py +++ b/geoprob_pipe/input_data/traject_normering.py @@ -14,17 +14,17 @@ def _data_from_metadata_table(app_settings: ApplicationSettings) -> Tuple[str, b df: DataFrame = read_sql_query("SELECT * FROM geoprob_pipe_metadata;", conn) conn.close() - traject_id = df[df["metadata_type"] == "traject_id"]["values"].values[0] - signaleringswaarde = int(df[df["metadata_type"] == "signaleringswaarde"]["values"].values[0]) - ondergrens = int(df[df["metadata_type"] == "ondergrens"]["values"].values[0]) - w = float(df[df["metadata_type"] == "w"]["values"].values[0]) - is_bovenrivierengebied = bool(int(df[df["metadata_type"] == "is_bovenrivierengebied"]["values"].values[0])) + traject_id = df[df["metadata_type"] == "traject_id"]["metadata_value"].values[0] + signaleringswaarde = int(df[df["metadata_type"] == "signaleringswaarde"]["metadata_value"].values[0]) + ondergrens = int(df[df["metadata_type"] == "ondergrens"]["metadata_value"].values[0]) + w = float(df[df["metadata_type"] == "w"]["metadata_value"].values[0]) + is_bovenrivierengebied = bool(int(df[df["metadata_type"] == "is_bovenrivierengebied"]["metadata_value"].values[0])) return traject_id, is_bovenrivierengebied, signaleringswaarde, ondergrens, w def _get_traject_length(app_settings: ApplicationSettings) -> float: - gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="dijktraject") + gdf_dijktraject: GeoDataFrame = read_file(app_settings.geopackage_filepath, layer="geom__dijktraject") gdf_dijktraject_geom = gdf_dijktraject.iloc[0].geometry if isinstance(gdf_dijktraject_geom, MultiLineString): assert gdf_dijktraject_geom.geoms.__len__() == 1 diff --git a/geoprob_pipe/results/__init__.py b/geoprob_pipe/results/__init__.py index c76001cd..16b27102 100644 --- a/geoprob_pipe/results/__init__.py +++ b/geoprob_pipe/results/__init__.py @@ -1,20 +1,28 @@ from __future__ import annotations + +import os +import sqlite3 +from datetime import datetime +from typing import TYPE_CHECKING + from pandas import DataFrame -from typing import TYPE_CHECKING, Optional + +from geoprob_pipe.results.alphas_and_physical_values import construct_df from geoprob_pipe.results.construct_dataframes import ( + calculate_df_beta_per_uittredepunt, combine_df_beta_per_limit_state, combine_df_beta_per_scenario_cp, - combine_df_beta_per_scenario_rp, combine_df_beta_per_scenario_final, - calculate_df_beta_per_uittredepunt, - construct_df_beta_wbi_vak, - construct_df_beta_window_vak, - construct_df_beta_scaled_vak, + combine_df_beta_per_scenario_rp, construct_df_beta_per_traject, + construct_df_beta_scaled_traject, + construct_df_beta_scaled_vak, + construct_df_beta_wbi_vak, construct_df_beta_window_traject, - construct_df_beta_scaled_traject) -from geoprob_pipe.results.alphas_and_physical_values import construct_df -import os + construct_df_beta_window_vak, +) +from geoprob_pipe.utils.sql_contents import write_dfs_to_gpkg + if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe @@ -31,7 +39,7 @@ def __init__(self, geoprob_pipe: GeoProbPipe): # Scenario calculations as a single Combine Project. self.df_beta_scenarios_final = combine_df_beta_per_scenario_final(geoprob_pipe.calc_results) # Worst result from scenario calculations, either combine project, reliability project or max of limit states. - self._df_alphas_influence_factors_and_physical_values: Optional[DataFrame] = None + self._df_alphas_influence_factors_and_physical_values = DataFrame() self.df_beta_uittredepunten = calculate_df_beta_per_uittredepunt( geoprob_pipe=geoprob_pipe, results=self ) @@ -63,6 +71,8 @@ def __init__(self, geoprob_pipe: GeoProbPipe): self.df_beta_scaled_traject = construct_df_beta_scaled_traject( geoprob_pipe=geoprob_pipe, results=self ) + self.df_alphas_influence_factors_and_physical_values() + self.export_to_gpkg() def df_alphas_influence_factors_and_physical_values( self, @@ -71,7 +81,7 @@ def df_alphas_influence_factors_and_physical_values( ) -> DataFrame: # Generate if not generated yet - if self._df_alphas_influence_factors_and_physical_values is None: + if self._df_alphas_influence_factors_and_physical_values.empty: self._df_alphas_influence_factors_and_physical_values = ( construct_df(self.geoprob_pipe)) @@ -114,6 +124,35 @@ def export_dir_traject(self) -> str: os.makedirs(path, exist_ok=True) return path + def export_to_gpkg(self): + """ + Voeg de overige resultaten toe aan de database voor inzicht en later + gebruik. + """ + conn = sqlite3.connect( + self.geoprob_pipe.input_data.app_settings.geopackage_filepath + ) + + tables = { + "result__physical_values": self._df_alphas_influence_factors_and_physical_values, + "result__vakken_WBI": self.df_beta_WBI_vakken, + "result__vakken_50m": self.df_beta_window50m_vakken, + "result__vakken_100m": self.df_beta_window100m_vakken, + "result__vakken_200m": self.df_beta_window200m_vakken, + "result__vakken_300m": self.df_beta_window300m_vakken, + "result__vakken_scaled": self.df_beta_scaled_vakken, + "result__traject": self.df_beta_traject, + "result__traject_50m": self.df_beta_window50m_traject, + "result__traject_100m": self.df_beta_window100m_traject, + "result__traject_200m": self.df_beta_window200m_traject, + "result__traject_300m": self.df_beta_window300m_traject, + "result__traject_scaled": self.df_beta_scaled_traject, + } + + write_dfs_to_gpkg(conn, tables) + + conn.close() + def export_results( self, bool_beta_limit_states: bool = True, @@ -125,6 +164,9 @@ def export_results( bool_beta_vakken: bool = True, bool_beta_alternative_vakken: bool = False, bool_beta_traject: bool = True): + """ + Exporteer resultaten naar excel op basis van instellingen. + """ # Results of limit state calculations if bool_beta_limit_states: @@ -193,3 +235,4 @@ def export_results( self.df_beta_scaled_traject.to_excel( excel_writer=os.path.join(self.export_dir_traject, "df_beta_scaled_traject.xlsx")) + \ No newline at end of file diff --git a/geoprob_pipe/results/assemblage/functions.py b/geoprob_pipe/results/assemblage/functions.py index c3db7dd3..84f060f2 100644 --- a/geoprob_pipe/results/assemblage/functions.py +++ b/geoprob_pipe/results/assemblage/functions.py @@ -17,7 +17,7 @@ def combine_series(list_pf: list[float]) -> Tuple[float, float]: doorsneden. :param list_pf: Lijst met faalkansen van de elementen. - :return bovengrens en ondergrens + :return: bovengrens en ondergrens """ # If empty @@ -28,7 +28,7 @@ def combine_series(list_pf: list[float]) -> Tuple[float, float]: ondergrens = max(list_pf) # We have to use Decimal for bovengrens - getcontext().prec = 30 + getcontext().prec = 100 # Because with small numbers (e-18 and smaller) it turns out that 1 - e-18 is rounded to one. Therefore, we have to # use Decimal with a lowered precision (we use up to e-30). We now first convert the necessary values to Decimal: one = Decimal(1) @@ -111,13 +111,15 @@ def window_collect(window_size: float, point_list: list[UittredepuntElement], bins_window = np.arange( m_van, m_tot, window_size ).tolist() - bins_window.append(m_tot) + if m_tot not in bins_window: + bins_window.append(m_tot) # add end of final window bin_cat: pd.Categorical = cast(pd.Categorical, pd.cut( list_m_value, bins=bins_window, right=False, - include_lowest=True + include_lowest=True, + duplicates="drop" )) df_vak = df_vak.assign(bin=bin_cat) df_bin = (df_vak.groupby("bin", observed=False)["pf"].max() @@ -125,8 +127,8 @@ def window_collect(window_size: float, point_list: list[UittredepuntElement], sum_pf, max_pf = combine_series(df_bin.to_list()) window_elements: List[WindowElement] = [] - bins_window.append(m_tot) # add end of final window - for i in range(len(bins_window)-2): + + for i in range(len(bins_window)-1): window_elements.append(WindowElement( m_van=bins_window[i], m_tot=bins_window[i+1], @@ -164,7 +166,7 @@ def scaled_collect( windows. """ from geoprob_pipe.results.assemblage.objects import WindowElement - if point_list.__len__() == 0: # Leeg element + if len(point_list) == 0: # Leeg element return 0.0, 0.0, [] fcn_list = [p.flow_chart_number for p in point_list] if min(fcn_list) == 11: @@ -214,7 +216,7 @@ def scaled_collect( length = seg_end - seg_start a = sel.a N_vak = bepaal_N_vak(length, a, dL) - pf = cast(float, sel.pf) * N_vak + pf = min(1, cast(float, sel.pf) * N_vak) # Limiet als pf zeer hoog is. pfs.append(pf) window_elements.append( WindowElement( diff --git a/geoprob_pipe/results/assemblage/objects.py b/geoprob_pipe/results/assemblage/objects.py index cb8f1ad5..37a1917b 100644 --- a/geoprob_pipe/results/assemblage/objects.py +++ b/geoprob_pipe/results/assemblage/objects.py @@ -25,13 +25,13 @@ def __post_init__(self): if self.pf is not None: if not (0.0 <= self.pf <= 1.0): - raise ValueError("pof moet tussen 0.0 en 1.0 liggen.") + raise ValueError(f"pof moet tussen 0.0 en 1.0 liggen. Huidige waarde is {self.pf}") - if self.beta is not None: - if not math.isfinite(self.beta): - raise ValueError("beta moet een eindige waarde zijn.") - if (-38.0 <= self.beta <= 38.0) is False: - raise ValueError("beta moet tussen -38.0 en 38.0 liggen.") + # if self.beta is not None: + # if not math.isfinite(self.beta): + # raise ValueError("beta moet een eindige waarde zijn.") + # if (-38.0 <= self.beta <= 38.0) is False: + # raise ValueError(f"beta moet tussen -38.0 en 38.0 liggen. Huidige waarde is {self.beta}") if self.pf is None and self.beta is not None: self.pf = float(stats.norm.cdf(-1.0 * self.beta)) @@ -56,13 +56,13 @@ def __post_init__(self): if self.pf is not None: if not (0.0 <= self.pf <= 1.0): - raise ValueError("pof moet tussen 0.0 en 1.0 liggen.") + raise ValueError(f"pof moet tussen 0.0 en 1.0 liggen. Huidige waarde is {self.pf}") - if self.beta is not None: - if not math.isfinite(self.beta): - raise ValueError("beta moet een eindige waarde zijn.") - if (-38.0 <= self.beta <= 38.0) is False: - raise ValueError("beta moet tussen -38.0 en 38.0 liggen.") + # if self.beta is not None: + # if not math.isfinite(self.beta): + # raise ValueError("beta moet een eindige waarde zijn.") + # if (-38.0 <= self.beta <= 38.0) is False: + # raise ValueError(f"beta moet tussen -38.0 en 38.0 liggen. Huidige waarde is {self.beta}") if self.pf is None and self.beta is not None: self.pf = float(stats.norm.cdf(-1.0 * self.beta)) @@ -121,7 +121,7 @@ def pf_max_dsn(self) -> Tuple[KansElement, KansElement]: pf_dsn = max(list_pf) # Bereken vak kans - pf_vak = self.N_vak * pf_dsn + pf_vak = min(1, self.N_vak * pf_dsn) # Limit naar 1 bij zeer hoge pof. return KansElement(pf=pf_dsn), KansElement(pf=pf_vak) @@ -131,7 +131,7 @@ def conv_max_dsn(self) -> bool: kansen geconvergeerd zijn, dan True, anders False. Want een niet geconvergeerde kans had de maximale faalkans kunnen zijn. """ list_conv = [cast(bool, dsn.converged) for dsn in self.dsn_list] - if False in list_conv: + if all(list_conv): return False return True diff --git a/geoprob_pipe/results/construct_dataframes.py b/geoprob_pipe/results/construct_dataframes.py index 63a0f42d..f9598452 100644 --- a/geoprob_pipe/results/construct_dataframes.py +++ b/geoprob_pipe/results/construct_dataframes.py @@ -1,18 +1,30 @@ from __future__ import annotations -from geoprob_pipe.utils.statistics import convert_failure_probability_to_beta + +import operator +from decimal import Decimal, getcontext +from functools import reduce +from typing import TYPE_CHECKING, List + +import numpy as np import pandas as pd -from geoprob_pipe.results.assemblage.objects import ( - UittredepuntElement, VakElement, TrajectElement) from pandas import DataFrame, concat -import numpy as np -from typing import TYPE_CHECKING, List + +from geoprob_pipe.results.assemblage.objects import ( + TrajectElement, + UittredepuntElement, + VakElement, +) +from geoprob_pipe.utils.statistics import convert_failure_probability_to_beta + if TYPE_CHECKING: - from geoprob_pipe.results import Results - from geoprob_pipe import GeoProbPipe - from geoprob_pipe.calculations.systems.base_objects.system_calculation import \ - SystemCalculation from probabilistic_library import DesignPoint + + from geoprob_pipe import GeoProbPipe + from geoprob_pipe.calculations.systems.base_objects.system_calculation import ( + SystemCalculation, + ) from geoprob_pipe.calculations.systems.build_and_run import CalcResult + from geoprob_pipe.results import Results def collect_df_beta_limit_state(calculation: SystemCalculation) -> DataFrame: @@ -20,7 +32,9 @@ def collect_df_beta_limit_state(calculation: SystemCalculation) -> DataFrame: def create_row(dp: DesignPoint, model_name): return { "uittredepunt_id": calculation.metadata["uittredepunt_id"], - "ondergrondscenario_id": calculation.metadata["ondergrondscenario_naam"], # TODO: id naar naam veranderen? + "ondergrondscenario_id": calculation.metadata[ + "ondergrondscenario_naam" + ], # TODO: id naar naam veranderen? "vak_id": calculation.metadata["vak_id"], "limit_state": model_name, "converged": dp.is_converged, @@ -32,49 +46,74 @@ def create_row(dp: DesignPoint, model_name): } rows = [] - for design_point, model in zip(calculation.results.dps_limit_states, calculation.setup.system_limit_states): + for design_point, model in zip( + calculation.results.dps_limit_states, + calculation.setup.system_limit_states, + ): rows.append(create_row(dp=design_point, model_name=model.__name__)) - df = DataFrame(rows).sort_values(by=["uittredepunt_id", "ondergrondscenario_id", "vak_id"]).reset_index(drop=True) + df = ( + DataFrame(rows) + .sort_values(by=["uittredepunt_id", "ondergrondscenario_id", "vak_id"]) + .reset_index(drop=True) + ) return df -def combine_df_beta_per_limit_state(calc_results: List[CalcResult]) -> DataFrame: +def combine_df_beta_per_limit_state( + calc_results: List[CalcResult], +) -> DataFrame: assert calc_results.__len__() > 0 - df = concat((result.df_limit_state for result in calc_results), ignore_index=True) + df = concat( + (result.df_limit_state for result in calc_results), ignore_index=True + ) return df def collect_df_beta_scenario_rp(calc: SystemCalculation) -> DataFrame: - return DataFrame([{ - "uittredepunt_id": calc.metadata["uittredepunt_id"], - "ondergrondscenario_id": calc.metadata["ondergrondscenario_naam"], # TODO: id naar naam veranderen? - "vak_id": calc.metadata["vak_id"], - "system_calculation": calc, - "converged": calc.results.dp_reliability.is_converged, - "beta": round(calc.results.dp_reliability.reliability_index, 3), - "failure_probability": calc.results.dp_reliability.probability_failure, - "convergence": calc.results.dp_reliability.convergence, - "total_model_runs": calc.results.dp_reliability.total_model_runs, - "total_iterations": calc.results.dp_reliability.total_iterations, - }]) + return DataFrame( + [ + { + "uittredepunt_id": calc.metadata["uittredepunt_id"], + "ondergrondscenario_id": calc.metadata[ + "ondergrondscenario_naam" + ], # TODO: id naar naam veranderen? + "vak_id": calc.metadata["vak_id"], + "system_calculation": calc, + "converged": calc.results.dp_reliability.is_converged, + "beta": round( + calc.results.dp_reliability.reliability_index, 3 + ), + "failure_probability": calc.results.dp_reliability.probability_failure, + "convergence": calc.results.dp_reliability.convergence, + "total_model_runs": calc.results.dp_reliability.total_model_runs, + "total_iterations": calc.results.dp_reliability.total_iterations, + } + ] + ) def collect_df_beta_scenario_cp(calc: SystemCalculation) -> DataFrame: - return DataFrame([{ - "uittredepunt_id": calc.metadata["uittredepunt_id"], - "ondergrondscenario_id": calc.metadata["ondergrondscenario_naam"], # TODO: id naar naam veranderen? - "vak_id": calc.metadata["vak_id"], - "system_calculation": calc, - "converged": calc.results.dp_combine.is_converged, - "beta": round(calc.results.dp_combine.reliability_index, 3), - "failure_probability": calc.results.dp_combine.probability_failure, - "convergence": calc.results.dp_combine.convergence, - "total_model_runs": calc.results.dp_combine.total_model_runs, - }]) + return DataFrame( + [ + { + "uittredepunt_id": calc.metadata["uittredepunt_id"], + "ondergrondscenario_id": calc.metadata[ + "ondergrondscenario_naam" + ], # TODO: id naar naam veranderen? + "vak_id": calc.metadata["vak_id"], + "system_calculation": calc, + "converged": calc.results.dp_combine.is_converged, + "beta": round(calc.results.dp_combine.reliability_index, 3), + "failure_probability": calc.results.dp_combine.probability_failure, + "convergence": calc.results.dp_combine.convergence, + "total_model_runs": calc.results.dp_combine.total_model_runs, + } + ] + ) def collect_df_beta_scenario_final(calc: SystemCalculation) -> DataFrame: - """ Converts a SystemCalculation-object to a single-row DataFrame with the final result of the scenario + """Converts a SystemCalculation-object to a single-row DataFrame with the final result of the scenario calculations. Because there are several calculation methods, and the preferred result also depends on convergence, we use the below flow chart to determine the final result. @@ -91,16 +130,25 @@ def collect_df_beta_scenario_final(calc: SystemCalculation) -> DataFrame: converged1: bool = calc.results.dp_combine.is_converged beta2: float = calc.results.dp_reliability.reliability_index converged2: bool = calc.results.dp_reliability.is_converged - beta3: float = max([dp.reliability_index for dp in calc.results.dps_limit_states]) - converged3: bool = all([dp.is_converged for dp in calc.results.dps_limit_states]) + beta3: float = max( + [dp.reliability_index for dp in calc.results.dps_limit_states] + ) + converged3: bool = all( + [dp.is_converged for dp in calc.results.dps_limit_states] + ) return_dict = { "uittredepunt_id": calc.metadata["uittredepunt_id"], - "ondergrondscenario_id": calc.metadata["ondergrondscenario_naam"], # TODO: id naar naam veranderen? + "ondergrondscenario_id": calc.metadata[ + "ondergrondscenario_naam" + ], # TODO: id naar naam veranderen? "vak_id": calc.metadata["vak_id"], "system_calculation": calc, - "beta1": beta1, "converged1": converged1, - "beta2": beta2, "converged2": converged2, - "beta3": beta3, "converged3": converged3, + "beta1": beta1, + "converged1": converged1, + "beta2": beta2, + "converged2": converged2, + "beta3": beta3, + "converged3": converged3, } # Flow chart step 1: Combine Project converged? @@ -126,20 +174,28 @@ def collect_df_beta_scenario_final(calc: SystemCalculation) -> DataFrame: return DataFrame([return_dict]) # Flow chart step 3: Separate Limit States all converged? - pof3: float = min([dp.probability_failure for dp in calc.results.dps_limit_states]) + pof3: float = min( + [dp.probability_failure for dp in calc.results.dps_limit_states] + ) if converged3: return_dict["method_used"] = "3: Max Limit States" return_dict["flow_chart_number"] = 3 return_dict["failure_probability"] = pof3 return_dict["beta"] = beta3 return_dict["converged"] = converged3 - return_dict["advise"] = ("Result is probably a conservative approximation.You could consider to find " - "convergence for the Combine or Reliability method.") + return_dict["advise"] = ( + "Result is probably a conservative approximation.You could consider to find " + "convergence for the Combine or Reliability method." + ) return DataFrame([return_dict]) # Flow chart step 4: B >= 8.0 (of all methods)? all_pofs: List[float] = [pof3, pof2, pof1] - all_methods: List[str] = ["3: Max Limit States", "2: Reliability Project", "1: Combine Project"] + all_methods: List[str] = [ + "3: Max Limit States", + "2: Reliability Project", + "1: Combine Project", + ] all_betas: List[float] = [beta3, beta2, beta1] index_max_pof = np.argmax(all_pofs) if beta1 >= 8.0 and beta2 >= 8.0 and beta3 >= 8.0: @@ -148,7 +204,9 @@ def collect_df_beta_scenario_final(calc: SystemCalculation) -> DataFrame: return_dict["failure_probability"] = all_pofs[index_max_pof] return_dict["beta"] = all_betas[index_max_pof] return_dict["converged"] = False - return_dict["advise"] = "Result is that positive, no reason to fine tune." + return_dict["advise"] = ( + "Result is that positive, no reason to fine tune." + ) return DataFrame([return_dict]) # No positive result @@ -157,30 +215,53 @@ def collect_df_beta_scenario_final(calc: SystemCalculation) -> DataFrame: return_dict["failure_probability"] = all_pofs[index_max_pof] return_dict["beta"] = all_betas[index_max_pof] return_dict["converged"] = False - return_dict["advise"] = "Consider fine tuning the calculation settings (of e.g. FORM, Importance Sampling, etc)." + return_dict["advise"] = ( + "Consider fine tuning the calculation settings (of e.g. FORM, Importance Sampling, etc)." + ) return DataFrame([return_dict]) -def combine_df_beta_per_scenario_rp(calc_results: List[CalcResult]) -> DataFrame: - df = concat((result.df_scenario_rp for result in calc_results), ignore_index=True) - df = df.sort_values(["uittredepunt_id", "ondergrondscenario_id", "vak_id"]).reset_index(drop=True) +def combine_df_beta_per_scenario_rp( + calc_results: List[CalcResult], +) -> DataFrame: + df = concat( + (result.df_scenario_rp for result in calc_results), ignore_index=True + ) + df = df.sort_values( + ["uittredepunt_id", "ondergrondscenario_id", "vak_id"] + ).reset_index(drop=True) return df -def combine_df_beta_per_scenario_cp(calc_results: List[CalcResult]) -> DataFrame: - df = concat((result.df_scenario_cp for result in calc_results), ignore_index=True) - df = df.sort_values(["uittredepunt_id", "ondergrondscenario_id", "vak_id"]).reset_index(drop=True) +def combine_df_beta_per_scenario_cp( + calc_results: List[CalcResult], +) -> DataFrame: + df = concat( + (result.df_scenario_cp for result in calc_results), ignore_index=True + ) + df = df.sort_values( + ["uittredepunt_id", "ondergrondscenario_id", "vak_id"] + ).reset_index(drop=True) return df -def combine_df_beta_per_scenario_final(calc_results: List[CalcResult]) -> DataFrame: - df = concat((result.df_scenario_final for result in calc_results), ignore_index=True) - df = df.sort_values(["uittredepunt_id", "ondergrondscenario_id", "vak_id"]).reset_index(drop=True) +def combine_df_beta_per_scenario_final( + calc_results: List[CalcResult], +) -> DataFrame: + df = concat( + (result.df_scenario_final for result in calc_results), + ignore_index=True, + ) + df = df.sort_values( + ["uittredepunt_id", "ondergrondscenario_id", "vak_id"] + ).reset_index(drop=True) return df -def calculate_df_beta_per_uittredepunt(geoprob_pipe: GeoProbPipe, results: Results) -> DataFrame: - """ Generates the DataFrame of the final result for the exit points. +def calculate_df_beta_per_uittredepunt( + geoprob_pipe: GeoProbPipe, results: Results +) -> DataFrame: + """Generates the DataFrame of the final result for the exit points. Because there is an automated decision-making in the scenario calculations (see flow chart over there), for the exit points the flow chart is extended below. @@ -196,36 +277,92 @@ def calculate_df_beta_per_uittredepunt(geoprob_pipe: GeoProbPipe, results: Resul df_beta_scenarios_final = results.df_beta_scenarios_final.copy(deep=True) - df = df_beta_scenarios_final.assign( - failure_probability=df_beta_scenarios_final.apply( - lambda row: row['failure_probability'] * geoprob_pipe.input_data.scenarios.scenario_kans( - vak_id=row['vak_id'], scenario_naam=row['ondergrondscenario_id'] - ), axis=1)).groupby('uittredepunt_id', as_index=False)[ - 'failure_probability'].sum() - df["beta"] = df["failure_probability"].apply(lambda failure_prob: convert_failure_probability_to_beta(failure_prob)) + # The sum() leads to failure probalilities of larger than 1. Due to very small + # values decimal is used. + getcontext().prec = 100 + one = Decimal(1) + + df = df_beta_scenarios_final.copy() + + def bereken_scenario_factor(row): + return geoprob_pipe.input_data.scenarios.scenario_kans( + vak_id=row["vak_id"], + scenario_naam=row["ondergrondscenario_id"], + ) + + df["scenario_factor"] = df.apply(bereken_scenario_factor, axis=1) + + df["failure_probability"] = ( + df["failure_probability"] * df["scenario_factor"] + ) + + def combine_probabilities(series): + c_pof = float( + one + - reduce( + operator.mul, + (one - Decimal(str(v)) for v in series), + one, + ) + ) + return c_pof + + df = df.groupby("uittredepunt_id", as_index=False).agg( + failure_probability=("failure_probability", combine_probabilities) + ) + + df["beta"] = df["failure_probability"].apply( + lambda failure_prob: convert_failure_probability_to_beta(failure_prob) + ) # Determine when uittredepunt is converged (when all scenarios are converged) - conv = df_beta_scenarios_final.groupby('uittredepunt_id', as_index=False)["converged"].all() + conv = df_beta_scenarios_final.groupby("uittredepunt_id", as_index=False)[ + "converged" + ].all() df_scen: pd.DataFrame = df.merge(conv, on="uittredepunt_id", how="left") # Determine uittredepunt flow_chart_number - flow_chart_number = df_beta_scenarios_final.groupby('uittredepunt_id', as_index=False)["flow_chart_number"].max() - df_scen = df_scen.merge(flow_chart_number, on="uittredepunt_id", how="left") - df_scen['advise'] = df_scen['flow_chart_number'].map({5: "Consider fine tuning on scenario-level."}).fillna("-") - df_scen['flow_chart_number'] = df_scen['flow_chart_number'].map({5: 11}).fillna(12) + flow_chart_number = df_beta_scenarios_final.groupby( + "uittredepunt_id", as_index=False + )["flow_chart_number"].max() + df_scen = df_scen.merge( + flow_chart_number, on="uittredepunt_id", how="left" + ) + df_scen["advise"] = ( + df_scen["flow_chart_number"] + .map({5: "Consider fine tuning on scenario-level."}) + .fillna("-") + ) + df_scen["flow_chart_number"] = ( + df_scen["flow_chart_number"].map({5: 11}).fillna(12) + ) # Add vak id back to it gdf_uittredepunten = geoprob_pipe.input_data.uittredepunten.gdf df_uittredepunten = gdf_uittredepunten[["uittredepunt_id", "vak_id"]] - df_scen = df_scen.merge(df_uittredepunten, left_on="uittredepunt_id", - right_on="uittredepunt_id") + df_scen = df_scen.merge( + df_uittredepunten, + left_on="uittredepunt_id", + right_on="uittredepunt_id", + ) - return df_scen[["uittredepunt_id", "vak_id", "converged", "beta", - "failure_probability", "advise", "flow_chart_number"]] + return df_scen[ + [ + "uittredepunt_id", + "vak_id", + "converged", + "beta", + "failure_probability", + "advise", + "flow_chart_number", + ] + ] -def _generate_point_list(geoprob_pipe: GeoProbPipe, results: Results) -> List[UittredepuntElement]: - """ Generates a list of all the points as a list `UittredepuntElement` objects. +def _generate_point_list( + geoprob_pipe: GeoProbPipe, results: Results +) -> List[UittredepuntElement]: + """Generates a list of all the points as a list `UittredepuntElement` objects. :param geoprob_pipe: GeoprobPipe object. :param results: @@ -235,11 +372,21 @@ def _generate_point_list(geoprob_pipe: GeoProbPipe, results: Results) -> List[Ui punt_gdf = geoprob_pipe.input_data.uittredepunten.gdf merge_df = pd.merge( - left=punt_df[["uittredepunt_id", "vak_id", "beta", "flow_chart_number", - "failure_probability", "converged", "advise"]], + left=punt_df[ + [ + "uittredepunt_id", + "vak_id", + "beta", + "flow_chart_number", + "failure_probability", + "converged", + "advise", + ] + ], right=punt_gdf[["uittredepunt_id", "metrering"]], - on="uittredepunt_id", how="left" - ) + on="uittredepunt_id", + how="left", + ) vakken_torun: List[int] = [] run_all: bool = False if geoprob_pipe.input_data.app_settings.to_run_vakken_ids: @@ -249,19 +396,28 @@ def _generate_point_list(geoprob_pipe: GeoProbPipe, results: Results) -> List[Ui dsn_list: List[UittredepuntElement] = [] for _, point in merge_df.iterrows(): if point["vak_id"] in vakken_torun or run_all: - dsn_list.append(UittredepuntElement( - pf=point["failure_probability"], - m_value=point["metrering"], - a=0.9, # TODO Haal deze vanuit Input Data via excel - converged=point["converged"], - flow_chart_number=point["flow_chart_number"], - advise=point["advise"] - )) + df_point = geoprob_pipe.df_expanded.loc[ + geoprob_pipe.df_expanded["parameter_name"] == "a_vak" + ] + a_point: float = df_point.query( + "uittredepunt_id == @point.uittredepunt_id" + )["parameter_input"].get("mean", 1.0) + dsn_list.append( + UittredepuntElement( + pf=point["failure_probability"], + m_value=point["metrering"], + a=a_point, + converged=point["converged"], + flow_chart_number=point["flow_chart_number"], + advise=point["advise"], + ) + ) return dsn_list -def _generate_element_list(geoprob_pipe: GeoProbPipe, results: Results - ) -> list[VakElement]: +def _generate_element_list( + geoprob_pipe: GeoProbPipe, results: Results +) -> list[VakElement]: """Generates a list of all elements in a traject as `VakElement` objects. :param geoprob_pipe: GeoprobPipe object. @@ -272,13 +428,22 @@ def _generate_element_list(geoprob_pipe: GeoProbPipe, results: Results punt_df = results.df_beta_uittredepunten punt_gdf = geoprob_pipe.input_data.uittredepunten.gdf - df = pd.merge( - left=punt_df[["uittredepunt_id", "vak_id", "beta", - "failure_probability", "converged", - "flow_chart_number", "advise"]], + df_utp = pd.merge( + left=punt_df[ + [ + "uittredepunt_id", + "vak_id", + "beta", + "failure_probability", + "converged", + "flow_chart_number", + "advise", + ] + ], right=punt_gdf[["uittredepunt_id", "metrering"]], - on="uittredepunt_id", how="left" - ) + on="uittredepunt_id", + how="left", + ) vakken_torun: List[int] = [] run_all: bool = False if geoprob_pipe.input_data.app_settings.to_run_vakken_ids: @@ -290,40 +455,75 @@ def _generate_element_list(geoprob_pipe: GeoProbPipe, results: Results for _, vak in vakken_gdf.iterrows(): if vak.id in vakken_torun or run_all: - df_vak = df.loc[df["vak_id"] == vak["id"]] + df_vak = df_utp.loc[df_utp["vak_id"] == vak["id"]] + + df_a = geoprob_pipe.df_expanded.loc[ + geoprob_pipe.df_expanded["parameter_name"] == "a_vak" + ] + a_vak: float = ( + df_a.query("vak_id == @vak.id") + .iloc[0]["parameter_input"] + .get("mean", 1.0) + ) + + df_dl = geoprob_pipe.df_expanded.loc[ + geoprob_pipe.df_expanded["parameter_name"] == "delta_length" + ] + dl_vak: float = ( + df_dl.query("vak_id == @vak.id") + .iloc[0]["parameter_input"] + .get("mean", 300.0) + ) + dsn_list = [] for _, point in df_vak.iterrows(): - dsn_list.append(UittredepuntElement( - pf=point["failure_probability"], - beta=point["beta"], - m_value=point["metrering"], - a=1.0, # TODO Haal deze vanuit Input Data via excel - converged=point["converged"], - flow_chart_number=point["flow_chart_number"], - advise=point["advise"])) - - element_list.append(VakElement( - id=vak["id"], - m_van=vak["m_start"], - m_tot=vak["m_end"], - a=1.0, # TODO Haal deze vanuit Input Data via excel - delta_length=300, - dsn_list=dsn_list - )) + a_point: float = ( + geoprob_pipe.df_expanded.loc[ + geoprob_pipe.df_expanded["parameter_name"] == "a_vak" + ] + .query("uittredepunt_id == @point.uittredepunt_id")[ + "parameter_input" + ] + .get("mean", 1.0) + ) + dsn_list.append( + UittredepuntElement( + pf=point["failure_probability"], + beta=point["beta"], + m_value=point["metrering"], + a=a_point, + converged=point["converged"], + flow_chart_number=point["flow_chart_number"], + advise=point["advise"], + ) + ) + + element_list.append( + VakElement( + id=vak["id"], + m_van=vak["m_start"], + m_tot=vak["m_end"], + a=a_vak, + delta_length=dl_vak, + dsn_list=dsn_list, + ) + ) return element_list def construct_df_beta_wbi_vak( - geoprob_pipe: GeoProbPipe, results: Results - ) -> pd.DataFrame: - """ Bepaalde de samengestelde faalkans per vak volgens de WBI-methode. + geoprob_pipe: GeoProbPipe, results: Results +) -> pd.DataFrame: + """Bepaalde de samengestelde faalkans per vak volgens de WBI-methode. :param geoprob_pipe: GeoProbPipe object voor data collectie. :param results: Results object voor data collectie. :return DataFrame: """ - element_list: list[VakElement] = _generate_element_list(geoprob_pipe=geoprob_pipe, results=results) + element_list: list[VakElement] = _generate_element_list( + geoprob_pipe=geoprob_pipe, results=results + ) vakken_list = [] for element in element_list: vakken_dict = { @@ -341,23 +541,26 @@ def construct_df_beta_wbi_vak( "beta_vak": element.pf_max_dsn[1].beta, "converged": element.conv_max_dsn, "flow_chart_number": element.flow_chart_number, - "advise": element.advise} + "advise": element.advise, + } vakken_list.append(vakken_dict) return pd.DataFrame(vakken_list) def construct_df_beta_window_vak( - geoprob_pipe: GeoProbPipe, results: Results, window_size: float - ) -> pd.DataFrame: - """ Bepaald de samengestelde faalkans per vak bij een window size. + geoprob_pipe: GeoProbPipe, results: Results, window_size: float +) -> pd.DataFrame: + """Bepaald de samengestelde faalkans per vak bij een window size. :param geoprob_pipe: GeoProbPipe object voor data collectie. :param window_size: :param results: Results object voor data collectie. :return DataFrame: """ - element_list: list[VakElement] = _generate_element_list(geoprob_pipe=geoprob_pipe, results=results) + element_list: list[VakElement] = _generate_element_list( + geoprob_pipe=geoprob_pipe, results=results + ) window_list = [] for element in element_list: for window in element.pf_window(window_size)[2]: @@ -376,24 +579,25 @@ def construct_df_beta_window_vak( "pf_vak": element.pf_window(window_size)[0].pf, "beta_vak": element.pf_window(window_size)[0].beta, "flow_chart_number": element.flow_chart_number, - "advise": element.advise - } + "advise": element.advise, + } window_list.append(window_dict) return pd.DataFrame(window_list) def construct_df_beta_scaled_vak( - geoprob_pipe: GeoProbPipe, results: Results - ) -> pd.DataFrame: - """ Bepaalde de samengestelde faalkans per vak op basis van de afstand tussen de uittredepunten. + geoprob_pipe: GeoProbPipe, results: Results +) -> pd.DataFrame: + """Bepaalde de samengestelde faalkans per vak op basis van de afstand tussen de uittredepunten. :param geoprob_pipe: GeoProbPipe object voor data collectie. :param results: Results object voor data collectie. :return DataFrame: """ - element_list = _generate_element_list(geoprob_pipe=geoprob_pipe, - results=results) + element_list = _generate_element_list( + geoprob_pipe=geoprob_pipe, results=results + ) window_list = [] for element in element_list: for window in element.pf_scaled[2]: @@ -412,16 +616,16 @@ def construct_df_beta_scaled_vak( "pf_vak": element.pf_scaled[0].pf, "beta_vak": element.pf_scaled[0].beta, "flow_chart_number": element.flow_chart_number, - "advise": element.advise - } + "advise": element.advise, + } window_list.append(window_dict) return pd.DataFrame(window_list) def construct_df_beta_per_traject( - geoprob_pipe: GeoProbPipe, results: Results - ) -> pd.DataFrame: + geoprob_pipe: GeoProbPipe, results: Results +) -> pd.DataFrame: """Bepaalde de samengestelde faalkans over het traject volgens de WBI-methode. :param geoprob_pipe: GeoProbPipe object voor data collectie. @@ -430,59 +634,64 @@ def construct_df_beta_per_traject( :return Dataframe: """ dsn_list = _generate_point_list(geoprob_pipe=geoprob_pipe, results=results) - vakken_list = _generate_element_list(geoprob_pipe=geoprob_pipe, results=results) + vakken_list = _generate_element_list( + geoprob_pipe=geoprob_pipe, results=results + ) - traject = TrajectElement(list_vakken=vakken_list, list_dsn=dsn_list, delta_length=300.0) + traject = TrajectElement( + list_vakken=vakken_list, list_dsn=dsn_list, delta_length=300.0 + ) traject_list = [ { "method": "WBI methode over traject", "upper_bound_pof": traject.pf_max_vak[0].pf, "lower_bound_beta": traject.pf_max_vak[0].beta, "lower_bound_pof": traject.pf_max_vak[1].pf, - "upper_bound_beta": traject.pf_max_vak[1].beta}, + "upper_bound_beta": traject.pf_max_vak[1].beta, + }, { "method": "Window 50m over traject", "upper_bound_pof": traject.pf_window(50.0)[0].pf, "lower_bound_beta": traject.pf_window(50.0)[0].beta, "lower_bound_pof": traject.pf_window(50.0)[1].pf, - "upper_bound_beta": traject.pf_window(50.0)[1].beta + "upper_bound_beta": traject.pf_window(50.0)[1].beta, }, { "method": "Window 100m over traject", "upper_bound_pof": traject.pf_window(100.0)[0].pf, "lower_bound_beta": traject.pf_window(100.0)[0].beta, "lower_bound_pof": traject.pf_window(100.0)[1].pf, - "upper_bound_beta": traject.pf_window(100.0)[1].beta + "upper_bound_beta": traject.pf_window(100.0)[1].beta, }, { "method": "Window 200m over traject", "upper_bound_pof": traject.pf_window(200.0)[0].pf, "lower_bound_beta": traject.pf_window(200.0)[0].beta, "lower_bound_pof": traject.pf_window(200.0)[1].pf, - "upper_bound_beta": traject.pf_window(200.0)[1].beta + "upper_bound_beta": traject.pf_window(200.0)[1].beta, }, { "method": "Window 300m over traject", "upper_bound_pof": traject.pf_window(300.0)[0].pf, "lower_bound_beta": traject.pf_window(300.0)[0].beta, "lower_bound_pof": traject.pf_window(300.0)[1].pf, - "upper_bound_beta": traject.pf_window(300.0)[1].beta + "upper_bound_beta": traject.pf_window(300.0)[1].beta, }, { "method": "Scaled over individual sections", "upper_bound_pof": traject.pf_scaled[0].pf, "lower_bound_beta": traject.pf_scaled[0].beta, "lower_bound_pof": traject.pf_scaled[1].pf, - "upper_bound_beta": traject.pf_scaled[1].beta + "upper_bound_beta": traject.pf_scaled[1].beta, }, ] return pd.DataFrame(traject_list) def construct_df_beta_window_traject( - geoprob_pipe: GeoProbPipe, results: Results, window_size: float - ) -> pd.DataFrame: - """ Bepaald de samengestelde faalkans voor het gehele traject bij een window size (i.p.v. vakken). + geoprob_pipe: GeoProbPipe, results: Results, window_size: float +) -> pd.DataFrame: + """Bepaald de samengestelde faalkans voor het gehele traject bij een window size (i.p.v. vakken). :param geoprob_pipe: GeoProbPipe object voor data collectie. :param results: Results object voor data collectie. @@ -490,9 +699,13 @@ def construct_df_beta_window_traject( :return DataFrame: """ dsn_list = _generate_point_list(geoprob_pipe=geoprob_pipe, results=results) - element_list = _generate_element_list(geoprob_pipe=geoprob_pipe, results=results) + element_list = _generate_element_list( + geoprob_pipe=geoprob_pipe, results=results + ) - traject = TrajectElement(list_vakken=element_list, list_dsn=dsn_list, delta_length=window_size) + traject = TrajectElement( + list_vakken=element_list, list_dsn=dsn_list, delta_length=window_size + ) window_list = [] for window in traject.pf_window(window_size=window_size)[2]: window_dict = { @@ -507,20 +720,22 @@ def construct_df_beta_window_traject( "a": 1.0, "delta_L": traject.delta_length, "N_vak": 1.0, - "pf_traject(sum)": traject.pf_window(window_size=window_size)[0].pf, + "pf_traject(sum)": traject.pf_window(window_size=window_size)[ + 0 + ].pf, "beta_traject": traject.pf_window(window_size=window_size)[0].beta, "flow_chart_number": window.flow_chart_number, - "advise": window.advise - } + "advise": window.advise, + } window_list.append(window_dict) return pd.DataFrame(window_list) def construct_df_beta_scaled_traject( - geoprob_pipe: GeoProbPipe, results: Results - ) -> pd.DataFrame: - """ Bepaalde de samengestelde faalkans voor het gehele traject op basis van de afstand tussen de uittredepunten + geoprob_pipe: GeoProbPipe, results: Results +) -> pd.DataFrame: + """Bepaalde de samengestelde faalkans voor het gehele traject op basis van de afstand tussen de uittredepunten i.p.v. de vakken. :param geoprob_pipe: GeoProbPipe object voor data collectie. @@ -528,8 +743,9 @@ def construct_df_beta_scaled_traject( :return DataFrame: """ dsn_list = _generate_point_list(geoprob_pipe=geoprob_pipe, results=results) - element_list = _generate_element_list(geoprob_pipe=geoprob_pipe, - results=results) + element_list = _generate_element_list( + geoprob_pipe=geoprob_pipe, results=results + ) traject = TrajectElement( list_vakken=element_list, list_dsn=dsn_list, delta_length=300.0 @@ -550,8 +766,8 @@ def construct_df_beta_scaled_traject( "pf_traject": traject.pf_scaled[0].pf, "beta_traject": traject.pf_scaled[0].beta, "flow_chart_number": window.flow_chart_number, - "advise": window.advise - } + "advise": window.advise, + } window_list.append(window_dict) return pd.DataFrame(window_list) diff --git a/geoprob_pipe/spatial/__init__.py b/geoprob_pipe/spatial/__init__.py index 421e47aa..5ce6b43f 100644 --- a/geoprob_pipe/spatial/__init__.py +++ b/geoprob_pipe/spatial/__init__.py @@ -1,9 +1,17 @@ from __future__ import annotations + from typing import TYPE_CHECKING -from geoprob_pipe.spatial.gdf_beta_limit_states import ( - get_gdf_beta_limit_states, get_gdf_beta_scenarios_final, get_gdf_beta_scenarios_cp, get_gdf_beta_scenarios_rp, - get_gdf_beta_uittredepunten) + from geopandas import GeoDataFrame + +from geoprob_pipe.spatial.gdf_beta_limit_states import ( + get_gdf_beta_limit_states, + get_gdf_beta_scenarios_cp, + get_gdf_beta_scenarios_final, + get_gdf_beta_scenarios_rp, + get_gdf_beta_uittredepunten, +) + if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe @@ -30,7 +38,7 @@ def get_gdf_beta_uittredepunten(self, export: bool = False) -> GeoDataFrame: @property def export_dir(self) -> str: - return self.geoprob_pipe.input_data.app_settings.workspace_dir + return str(self.geoprob_pipe.input_data.app_settings.workspace_dir) def export_geopackage(self): self.get_gdf_beta_limit_states(export=True) diff --git a/geoprob_pipe/spatial/gdf_beta_limit_states.py b/geoprob_pipe/spatial/gdf_beta_limit_states.py index 2638487c..552cf638 100644 --- a/geoprob_pipe/spatial/gdf_beta_limit_states.py +++ b/geoprob_pipe/spatial/gdf_beta_limit_states.py @@ -22,7 +22,7 @@ def get_gdf_beta_limit_states(geoprob_pipe: GeoProbPipe, export: bool = False) - gdf = _get_uittredepunten_gdf_beta_results(geoprob_pipe, geoprob_pipe.results.df_beta_limit_states) if export: gdf.to_file(geoprob_pipe.input_data.app_settings.geopackage_filepath, - layer="beta_limit_states", driver="GPKG", mode="w") + layer="result__limit_states", driver="GPKG", mode="w") return gdf @@ -31,7 +31,7 @@ def get_gdf_beta_scenarios_final(geoprob_pipe: GeoProbPipe, export: bool = False if export: gdf.to_file( geoprob_pipe.input_data.app_settings.geopackage_filepath, - layer="beta_scenarios_final", driver="GPKG", mode="w") + layer="result__scenarios_final", driver="GPKG", mode="w") return gdf @@ -40,7 +40,7 @@ def get_gdf_beta_scenarios_cp(geoprob_pipe: GeoProbPipe, export: bool = False) - if export: gdf.to_file( geoprob_pipe.input_data.app_settings.geopackage_filepath, - layer="beta_scenarios_cp", driver="GPKG", mode="w") + layer="result__scenarios_cp", driver="GPKG", mode="w") return gdf @@ -49,7 +49,7 @@ def get_gdf_beta_scenarios_rp(geoprob_pipe: GeoProbPipe, export: bool = False) - if export: gdf.to_file( geoprob_pipe.input_data.app_settings.geopackage_filepath, - layer="beta_scenarios_rp", driver="GPKG", mode="w") + layer="result__scenarios_rp", driver="GPKG", mode="w") return gdf @@ -58,5 +58,5 @@ def get_gdf_beta_uittredepunten(geoprob_pipe: GeoProbPipe, export: bool = False) if export: gdf.to_file( geoprob_pipe.input_data.app_settings.geopackage_filepath, - layer="beta_uittredepunten", driver="GPKG", mode="w") + layer="result__uittredepunten", driver="GPKG", mode="w") return gdf diff --git a/geoprob_pipe/utils/df_validation/dataframe.py b/geoprob_pipe/utils/df_validation/dataframe.py index 4501d415..ff67a813 100644 --- a/geoprob_pipe/utils/df_validation/dataframe.py +++ b/geoprob_pipe/utils/df_validation/dataframe.py @@ -17,15 +17,15 @@ class DataFrameValidation: def __init__( self, df: DataFrame, label: str, - columns_validations: List[ColumnValidation], required_columns: List[str] = None, + columns_validations: List[ColumnValidation], required_columns: List[str] = [], ): self.label: str = label self.df: DataFrame = df - self.required_columns: Optional[List[str]] = required_columns + self.required_columns: List[str] = required_columns self.columns_validations: List[ColumnValidation] = columns_validations # Placeholders - self.df_failures: Optional[DataFrame] = None + self.df_failures: DataFrame = DataFrame() def _run_required_columns(self) -> DataFrame: failure_rows = [] diff --git a/geoprob_pipe/utils/df_validation/requirements.py b/geoprob_pipe/utils/df_validation/requirements.py index 3499ed83..66d0354a 100644 --- a/geoprob_pipe/utils/df_validation/requirements.py +++ b/geoprob_pipe/utils/df_validation/requirements.py @@ -34,11 +34,13 @@ def __init__( def is_not_null(s: Series) -> Series: - return s.notna() + s_num = to_numeric(s, errors="coerce") + return s_num.notna() def is_null(s: Series) -> Series: - return ~s.notna() + s_num = to_numeric(s, errors="coerce") + return ~s_num.notna() def is_in(values: List): @@ -60,8 +62,10 @@ def is_integer(s: Series) -> Series: def is_whole_number(s: Series) -> Series: - return s.notna() & (s % 1 == 0) + s_num = to_numeric(s, errors="coerce") # De tweede test breekt als er iets anders dan een getal in staan. + return s_num.notna() & (s_num % 1 == 0) def is_numeric(s: Series) -> Series: - return to_numeric(s, errors="coerce").notna() + s_num = to_numeric(s, errors="coerce") + return s_num.notna() diff --git a/geoprob_pipe/utils/sql_contents.py b/geoprob_pipe/utils/sql_contents.py new file mode 100644 index 00000000..efc8c95c --- /dev/null +++ b/geoprob_pipe/utils/sql_contents.py @@ -0,0 +1,138 @@ +import sqlite3 +from datetime import datetime + +import pandas as pd + + +def _create_table_with_pk(conn, table_name, df): + """ + Maak de tabel aan met de juiste kollomen en een primary key. + + :param conn: Database connector + :param table_name: Naam van de toe te voegen tabel + :param df: DataFrame die wordt toegevoegd + """ + cursor = conn.cursor() + + # Verwijder bestaande tabel + cursor.execute(f'DROP TABLE IF EXISTS "{table_name}"') + + # SQLite types bepalen + cols_sql = [] + for col, dtype in df.dtypes.items(): + if "int" in str(dtype): + sql_type = "INTEGER" + elif "float" in str(dtype): + sql_type = "REAL" + else: + sql_type = "TEXT" + + cols_sql.append(f'"{col}" {sql_type}') + + cols_sql = ", ".join(cols_sql) + + # Setup primary key + cursor.execute(f''' + CREATE TABLE "{table_name}" ( + fid INTEGER PRIMARY KEY AUTOINCREMENT, + {cols_sql} + ) + ''') + +def _register_gpkg_data_columns(conn, table_name): + """ + Voeg de kolommen toe aan de gpkg_data_columns vanuit de ingevoerde tabel. + + :param conn: Database connector + :param table_name: Naam van toegevoegde tabel. + """ + cursor = conn.cursor() + + # Haal kolomnamen op via SQLite pragma + cursor.execute(f"PRAGMA table_info('{table_name}')") + columns = cursor.fetchall() + + for col in columns: + col_name = col[1] # tweede veld = column name + + cursor.execute( + """ + INSERT OR REPLACE INTO gpkg_data_columns ( + table_name, + column_name, + name, + title, + description, + mime_type, + constraint_name + ) + VALUES (?, ?, ?, ?, '', NULL, NULL) + """, + ( + table_name, + col_name, + col_name, + col_name, + ), + ) + +def write_dfs_to_gpkg(conn: sqlite3.Connection, tables: dict[str, pd.DataFrame]): + """ + Schrijf de pandas DataFrames naar de databese met een fid primary key en + voeg deze toe aan de gpkg_data_columns zodat deze in gis-software kunnen + worden geopend. + + :param conn: Databse connector + :param tables: dict met de atbelnamen en dataframes + """ + cursor = conn.cursor() + + # Maak gpkg_data_columns tabel als deze nog niet bestaat. + cursor.execute(""" + CREATE TABLE IF NOT EXISTS gpkg_data_columns ( + table_name TEXT NOT NULL, + column_name TEXT NOT NULL, + name TEXT, + title TEXT, + description TEXT, + mime_type TEXT, + constraint_name TEXT, + PRIMARY KEY (table_name, column_name) + ) + """) + + for table_name, df in tables.items(): + if df.empty: # Sla lege df over, dit breekt de SQL + continue + + _create_table_with_pk(conn, table_name, df) + + # Voeg de df toe aan de tabel, met append anders overschijft het de primary key. + df.to_sql(table_name, conn, if_exists="append", index=False) + + cursor.execute( + "DELETE FROM gpkg_contents WHERE table_name = ?", (table_name,) + ) + + # Registreren in gpkg_contents + cursor.execute( + """ + INSERT INTO gpkg_contents ( + table_name, + data_type, + identifier, + description, + last_change + ) + VALUES (?, 'attributes', ?, '', ?) + """, + ( + table_name, + table_name, + datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + ), + ) + + _register_gpkg_data_columns(conn, table_name) + + conn.commit() diff --git a/geoprob_pipe/utils/update_metadata.py b/geoprob_pipe/utils/update_metadata.py index 4626aba4..06dfbb85 100644 --- a/geoprob_pipe/utils/update_metadata.py +++ b/geoprob_pipe/utils/update_metadata.py @@ -17,8 +17,8 @@ def _to_text(v): def _upsert_metadata(conn, table, metadata_type, value): - sql_update = f'UPDATE {table} SET "values" = ? WHERE metadata_type = ?' - sql_insert = f'INSERT INTO {table} (metadata_type, "values") VALUES (?, ?)' + sql_update = f'UPDATE {table} SET metadata_value = ? WHERE metadata_type = ?' + sql_insert = f'INSERT INTO {table} (metadata_type, metadata_value) VALUES (?, ?)' with conn: # transaction cur = conn.execute(sql_update, (value, metadata_type)) @@ -43,19 +43,19 @@ def update_metadata(geoprob_pipe: GeoProbPipe): records = [ {"metadata_type": "last_calculation_run_datetime", - "values": geoprob_pipe.input_data.app_settings.datetime_stamp}, + "metadata_value": geoprob_pipe.input_data.app_settings.datetime_stamp}, {"metadata_type": "last_calculation_rub_in_seconds", - "values": geoprob_pipe.time_diff.total_seconds()}, + "metadata_value": geoprob_pipe.time_diff.total_seconds()}, {"metadata_type": "last_calculation_run_vakken_to_run", - "values": ran_vakken_ids}, + "metadata_value": ran_vakken_ids}, {"metadata_type": "last_calculation_run_nr_of_calculations", - "values": n_calcs}, + "metadata_value": n_calcs}, {"metadata_type": "last_calculation_run_nr_of_uittredepunten", - "values": n_points}, + "metadata_value": n_points}, {"metadata_type": "last_calculation_run_nr_of_vakken", - "values": n_vakken}, + "metadata_value": n_vakken}, {"metadata_type": "last_calculation_run_percentage_of_uittredepunten", - "values": ratio_points * 100} + "metadata_value": ratio_points * 100} ] conn = sqlite3.connect(gpkg_path) @@ -63,6 +63,6 @@ def update_metadata(geoprob_pipe: GeoProbPipe): for r in records: _upsert_metadata(conn=conn, table=table, metadata_type=r["metadata_type"], - value=_to_text(r["values"])) + value=_to_text(r["metadata_value"])) conn.commit() conn.close() diff --git a/geoprob_pipe/visualizations/graphs/__init__.py b/geoprob_pipe/visualizations/graphs/__init__.py index 50b90198..4a98c861 100644 --- a/geoprob_pipe/visualizations/graphs/__init__.py +++ b/geoprob_pipe/visualizations/graphs/__init__.py @@ -7,7 +7,7 @@ from geoprob_pipe.visualizations.graphs.physical_values_along_levee import physical_values_buitenwaterstand_and_top_zand from geoprob_pipe.visualizations.graphs.invloedsfactoren import invloedsfactoren from geoprob_pipe.visualizations.graphs.phreatic_waterline import phreatic_waterline -from geoprob_pipe.visualizations.graphs.overview_alpha import overview_alpha +from geoprob_pipe.visualizations.graphs.overview_alpha import OverviewAlpha from geoprob_pipe.visualizations.graphs.river_waterlevel import river_waterlevel from typing import TYPE_CHECKING from plotly.graph_objects import Figure as PlotlyFigure @@ -59,7 +59,7 @@ def phreatic_waterline(self) -> PlotlyFigure: return phreatic_waterline(self.geoprob_pipe, export=False) def overview_alpha(self) -> PlotlyFigure: - return overview_alpha(self.geoprob_pipe, export=False) + return OverviewAlpha(self.geoprob_pipe, export=False).fig def river_waterlevel(self) -> PlotlyFigure: return river_waterlevel(self.geoprob_pipe, export=False) @@ -71,5 +71,5 @@ def export_graphs(self): self.physical_values_buitenwaterstand_and_top_zand(export=True) self.invloedsfactoren(export=True) phreatic_waterline(self.geoprob_pipe, export=True) - overview_alpha(self.geoprob_pipe, export=True) + OverviewAlpha(self.geoprob_pipe, export=True) river_waterlevel(self.geoprob_pipe, export=True) diff --git a/geoprob_pipe/visualizations/graphs/assemblage_icicle.py b/geoprob_pipe/visualizations/graphs/assemblage_icicle.py index 58c4ba0d..653a17cc 100644 --- a/geoprob_pipe/visualizations/graphs/assemblage_icicle.py +++ b/geoprob_pipe/visualizations/graphs/assemblage_icicle.py @@ -1,10 +1,11 @@ from __future__ import annotations + import os -import plotly.graph_objects as go -import pandas as pd from typing import TYPE_CHECKING, Dict, List, cast -from pandas import Series, DataFrame +import pandas as pd +import plotly.graph_objects as go +from pandas import DataFrame, Series if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe @@ -12,6 +13,14 @@ class IciclePlot: def __init__(self, geoprob_pipe: GeoProbPipe, export: bool = False): + """ + Class voor het maken en exporteren van een Icicle plot van de assemblage. + Hierin kan worden gezien hoe de faalkansen en beta's tot stand komen + vanuit onderliggende elementen. + + :param geoprob_pipe: Hoofdobject van de applicatie. + :param export: Of het figuur geëxporteerd wordt, defaults to False. + """ self.geoprob_pipe: GeoProbPipe = geoprob_pipe self.export: bool = export self.fig = go.Figure() @@ -21,9 +30,14 @@ def __init__(self, geoprob_pipe: GeoProbPipe, export: bool = False): self._optionally_export() def _setup_df(self): + """ + Set de DataFrame op voor de icicle door de verschillende lagen stap + voor stap samen te voegen. De `id` kolom functioneert als key voor het + koppelen. + """ # Per element parent_name, label, pf, beta - df_traject = self.geoprob_pipe.results.df_beta_traject - df_vakken = self.geoprob_pipe.results.df_beta_WBI_vakken + df_traject = self.geoprob_pipe.results.df_beta_traject.copy() + df_vakken = self.geoprob_pipe.results.df_beta_WBI_vakken.copy() df_vakken = df_vakken.rename(columns={"beta_dsn": "beta", "pf_dsn(max)": "pf"}) @@ -31,9 +45,9 @@ def _setup_df(self): mask_vakken = df_vakken["pf"] != 0 df_vakken = df_vakken[mask_vakken].copy() - df_utp = self.geoprob_pipe.results.df_beta_uittredepunten - df_scen = self.geoprob_pipe.results.df_beta_scenarios_final - df_lim = self.geoprob_pipe.results.df_beta_limit_states + df_utp = self.geoprob_pipe.results.df_beta_uittredepunten.copy() + df_scen = self.geoprob_pipe.results.df_beta_scenarios_final.copy() + df_lim = self.geoprob_pipe.results.df_beta_limit_states.copy() # traject mask_traject = df_traject["method"] == "Sum of vakken" @@ -148,6 +162,12 @@ def _setup_df(self): self.df_icicle = df_icicle def _beta_to_color(self, beta: float) -> str: + """ + Vindt de juiste riskeer kleuren bij de betawaardes op basis van de norm. + + :param beta: beta waarde om kleur van te bepalen. + :return: rgba waardes van de juiste kleur als string. + """ cg: Dict[str, List[float]] = self.geoprob_pipe.input_data.traject_normering.riskeer_categorie_grenzen # labels = ["+III", "+II", "+I", "0", "-I", "-II", "-III"] colors = [ @@ -161,9 +181,9 @@ def _beta_to_color(self, beta: float) -> str: ] # Limit beta to range for colors if beta < 2: - return "rgba(30,141,41,0.6)" - elif beta > 20: return "rgba(177,33,38,0.6)" + elif beta > 20: + return "rgba(30,141,41,0.6)" else: for i, grens in enumerate(cg): beta_min, beta_max = cg[grens] @@ -173,6 +193,9 @@ def _beta_to_color(self, beta: float) -> str: return "rgba(128,128,128,0.6)" def _plot(self): + """ + Maak de icicle-plot op basis van de ongezette dataframe. + """ df = self.df_icicle mask_vakken = df["parent_name"].eq("1") @@ -220,6 +243,9 @@ def _update_layout(self): ) def _optionally_export(self): + """ + Maak de html van de plot met ingebouwde scroll bar. + """ if self.export: path = self.geoprob_pipe.visualizations.graphs.export_dir diff --git a/geoprob_pipe/visualizations/graphs/betrouwbaarheidsindex.py b/geoprob_pipe/visualizations/graphs/betrouwbaarheidsindex.py index a28a4720..bfc1f16e 100644 --- a/geoprob_pipe/visualizations/graphs/betrouwbaarheidsindex.py +++ b/geoprob_pipe/visualizations/graphs/betrouwbaarheidsindex.py @@ -1,10 +1,13 @@ from __future__ import annotations -from pandas import DataFrame, merge, Series -import numpy as np + import os from datetime import datetime -import plotly.graph_objects as go from typing import TYPE_CHECKING + +import numpy as np +import plotly.graph_objects as go +from pandas import DataFrame, Series, merge + if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe @@ -85,6 +88,12 @@ def _add_beta_per_uittredepunt_indication_below_plotting_range( class GraphBetaValuesSingleInteractive: def __init__(self, geoprob_pipe: GeoProbPipe, export: bool = True): + """ + Class voor het maken van de betrouwbaarheidsindex plot. + + :param geoprob_pipe: Hoofdobject van de applicatie. + :param export: Of de figuur geëxporteerd moet worden, defaults to True + """ # Logic self.geoprob_pipe = geoprob_pipe @@ -112,6 +121,9 @@ def __init__(self, geoprob_pipe: GeoProbPipe, export: bool = True): self._optionally_export(export=export) def _add_backgrond(self): + """ + Voeg de achtergrond kleuren toe op basis van de normwaarden. + """ # Oude categorie grenzen # cg = (self.geoprob_pipe.input_data.traject_normering # .beta_categorie_grenzen) @@ -207,6 +219,11 @@ def _add_backgrond(self): )) def _add_beta_per_scenario(self): + """ + Voeg de traces van de scenarios toe aan de plot, inclusief extra + symbolen voor punten die buiten de range vallen. En niet geconvergeerde + resultaten in een andere kleur. + """ df_beta_scenarios_final = self.geoprob_pipe.results.df_beta_scenarios_final df_for_graph: DataFrame = merge( left=df_beta_scenarios_final[[ @@ -276,6 +293,11 @@ def _add_beta_per_scenario(self): "Metrering: %{customdata[2]}"))) def _add_beta_per_uittredepunt(self): + """ + Voeg de traces van de uittredepunten toe aan de plot, inclusief extra + symbolen voor punten die buiten de range vallen. En niet geconvergeerde + resultaten in een andere kleur. + """ # Gather results to plot df_results_uittredepunten = self.geoprob_pipe.results.df_beta_uittredepunten @@ -309,6 +331,11 @@ def _add_beta_per_uittredepunt(self): name=name, color=color, value=value) def _add_beta_per_vak(self): + """ + Voeg de traces van de vakken toe aan de plot, inclusief extra + symbolen voor punten die buiten de range vallen. En niet geconvergeerde + resultaten in een andere kleur. + """ # Gather data df_results_vakken = self.geoprob_pipe.results.df_beta_WBI_vakken df_results_vakken = df_results_vakken.rename( @@ -381,6 +408,9 @@ def _add_beta_per_vak(self): first = False def _add_beta_per_traject(self): + """ + Voeg de trace van het traject toe aan de plot. + """ df = self.geoprob_pipe.results.df_beta_traject df = df[df["method"] == 'WBI methode over traject'] beta_traject: float = df["lower_bound_beta"].iloc[0] @@ -400,6 +430,10 @@ def _add_beta_per_traject(self): )) def _update_layout(self): + """ + Voeg de lay-out van de plot toe inclusief knoppen om de lijnen en vak + annotaties aan en uit te zetten. + """ if self.verder_rekenen_geadviseerd: self.fig.add_trace(go.Scatter( x=[0], y=[0], name="Verder rekenen geadviseerd", showlegend=True, @@ -427,8 +461,8 @@ def _update_layout(self): legend=dict( yanchor="top", y=0.99, - xanchor="left", - x=0.01 + xanchor="right", + x=-0.01 ), annotations=annotation, shapes=self.vak_lines diff --git a/geoprob_pipe/visualizations/graphs/hfreq.py b/geoprob_pipe/visualizations/graphs/hfreq.py index b5462860..d7d334f1 100644 --- a/geoprob_pipe/visualizations/graphs/hfreq.py +++ b/geoprob_pipe/visualizations/graphs/hfreq.py @@ -21,7 +21,7 @@ def __init__(self, geoprob_pipe: GeoProbPipe, export: bool = False): # Check if there are fragility curves referenced: conn = sqlite3.connect(geoprob_pipe.input_data.app_settings.geopackage_filepath) - df_parameter_invoer = read_sql("SELECT * FROM parameter_invoer;", conn) + df_parameter_invoer = read_sql("SELECT * FROM data__excel_parameter_invoer;", conn) fragility_values_refs = df_parameter_invoer['fragility_values_ref'].unique() conn.close() if (fragility_values_refs.__len__() == 0 or @@ -148,7 +148,7 @@ def _add_overschrijdingsfrequentielijnen(self): for index, fragility_values_ref in enumerate(self.df_fragility_ref_data["fragility_values_ref"].unique()): # Collect data for the graph - fragility_values, _, uittredepunten, uittredepunten_multiline = self._get_fragility_values_ref_data( + fragility_values, _, _, uittredepunten_multiline = self._get_fragility_values_ref_data( ref=fragility_values_ref) levels = [item.x for item in fragility_values] self.max_level = max(self.max_level, max(levels)) @@ -219,10 +219,10 @@ def _update_layout(self): title=f"Overschrijdingsfrequentielijnen voor alle HydraNL locaties
" f"Traject {self.geoprob_pipe.input_data.traject_normering.traject_id}", xaxis=dict( - title=f"Waterstand (m+NAP)", type='linear', showgrid=True, gridwidth=0.5, gridcolor="gray", + title="Waterstand (m+NAP)", type='linear', showgrid=True, gridwidth=0.5, gridcolor="gray", range=[self.min_level - xaxis_add, self.max_level + xaxis_add]), yaxis=dict( - title=f"Overschrijdingsfrequentie (log-schaal)", type='log', showgrid=True, tickformat=".0e", + title="Overschrijdingsfrequentie (log-schaal)", type='log', showgrid=True, tickformat=".0e", gridwidth=1.0, tickvals=y_ticks, ticktext=y_ticks_text, tickmode='array', gridcolor="gray", minor=dict( showgrid=True, dtick="D1", gridwidth=0.5, gridcolor='rgb(199, 197, 193)'))) diff --git a/geoprob_pipe/visualizations/graphs/invloedsfactoren.py b/geoprob_pipe/visualizations/graphs/invloedsfactoren.py index 3c012a2d..63cc04a0 100644 --- a/geoprob_pipe/visualizations/graphs/invloedsfactoren.py +++ b/geoprob_pipe/visualizations/graphs/invloedsfactoren.py @@ -125,7 +125,7 @@ def _get_data(geoprob_pipe: GeoProbPipe) -> DataFrame: def _plot_data(geoprob_pipe: GeoProbPipe, df_invloedsfactoren: DataFrame) -> Figure: fig = Figure() gdf_vakindeling: GeoDataFrame = read_file( - geoprob_pipe.input_data.app_settings.geopackage_filepath, layer="vakindeling") + geoprob_pipe.input_data.app_settings.geopackage_filepath, layer="geom__vakindeling") vakken = {row['id']: row["naam"] for index, row in gdf_vakindeling.iterrows()} picked_colors = {} added_to_legend = [] diff --git a/geoprob_pipe/visualizations/graphs/overview_alpha.py b/geoprob_pipe/visualizations/graphs/overview_alpha.py index fefb4fd9..bcd2e13f 100644 --- a/geoprob_pipe/visualizations/graphs/overview_alpha.py +++ b/geoprob_pipe/visualizations/graphs/overview_alpha.py @@ -1,183 +1,227 @@ from __future__ import annotations + import os -import pandas as pd +from typing import TYPE_CHECKING + import plotly.graph_objects as go -from plotly.subplots import make_subplots from pandas import merge -from geoprob_pipe.calculations.systems.mappers.initial_input import INITIAL_INPUT_MAPPER -from typing import TYPE_CHECKING +from plotly.subplots import make_subplots +from plotly.graph_objects import Figure + +from geoprob_pipe.calculations.systems.mappers.initial_input import ( + INITIAL_INPUT_MAPPER, +) from geoprob_pipe.cmd_app.utils.misc import get_geohydrological_model if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe -def overview_alpha(geoprob_pipe: GeoProbPipe, export: bool = False): - - model_string = get_geohydrological_model(app_settings=geoprob_pipe.input_data.app_settings) - initial_input_mapper = INITIAL_INPUT_MAPPER[model_string]['input'] - - # Get data for graphing - df = geoprob_pipe.results.df_alphas_influence_factors_and_physical_values( - filter_deterministic=False, filter_derived=False) - df = df[["uittredepunt_id", "ondergrondscenario_id", "vak_id", "variable", "distribution_type", "physical_value"]] - - gdf_uittredepunten = geoprob_pipe.input_data.uittredepunten.gdf - df = merge(df, gdf_uittredepunten[["uittredepunt_id", "metrering"]], on="uittredepunt_id", how="left") - - # Determine scenario order per uittredepunt_id - df["scenario_order"] = ( - df.groupby("uittredepunt_id")["ondergrondscenario_id"].transform(lambda x: pd.factorize(x)[0] + 1)) - scenario_orders = sorted(df["scenario_order"].unique()) - - # List of all alphas that can be shown - parameters: list[str] = list(df["variable"].unique()) - # Add distribution type - dist_types = {} - for param in parameters: - dist_type = df.loc[ - df["variable"] == param, "distribution_type"].unique()[0] - dist_types.update({param: dist_type}) - # Add units - unit_lookup = {item["name"]: item["unit"] for item in initial_input_mapper} - param_units = {} - for param in parameters: - try: - param_units.update({param: str(unit_lookup[param]).strip("[]")}) - except KeyError: - param_units.update({param: "?"}) - - # Add parameter units missing in DUMMY_INPUT by hand. - param_units.update({ - "L_kwelweg": "m", - "L_voorland": "m", - "W_voorland": "s/m", - "buitenwaterstand_gemiddeld": "m+NAP", - "d_deklaag": "m", - "dh_c": "m", - "dh_red": "m", - "dphi_c_u": "m+NAP", - "h_exit": "m+NAP", - "i_exit": "-", - "k_wvp": "m/dag", - "lambda_voorland": "m", - "phi_exit": "m+NAP", - "phi_exit_gemiddeld": "m+NAP", - "r_exit": "-", - "z_combin": "-", - "z_h": "-", - "z_p": "-", - "z_u": "-" - }) - - # Create subplots - fig = make_subplots( - rows=len(parameters), - cols=1, - shared_xaxes=False, - subplot_titles=parameters - ) - - # Add a button for each ondergrondscenario - buttons = [] - - # Add scatter plot per scenario - for i, scen_order in enumerate(scenario_orders): - df_case = df[df["scenario_order"] == scen_order] - - for row_idx, param in enumerate(parameters, start=1): - df_param = df_case[df_case["variable"] == param] - fig.add_trace( - go.Scatter( - x=df_param["metrering"], - y=df_param["physical_value"], - mode="markers", - marker=dict(color="black", symbol="x", size=5), - name=param, - visible=(i == 0) - ), - row=row_idx, col=1 - ) - fig.update_xaxes(showgrid=True, tickangle=90, - row=row_idx, col=1) - fig.update_yaxes(showgrid=True, - title_text=f"{param} [{param_units[param]}]" - + f"
({dist_types[param]})", - row=row_idx, col=1) - - total_traces = len(scenario_orders) * len(parameters) - - # Determine which scenario is visible - vis = [False] * total_traces - vis[i*len(parameters):(i+1)*len(parameters)] = [True] * len(parameters) - - buttons.append(dict( - label=f"Scenario {scen_order}", - method="update", - args=[ - {"visible": vis}, - {"title": f"Overview of parameters for Scenario {scen_order}"} +class OverviewAlpha: + def __init__(self, geoprob_pipe: GeoProbPipe, export: bool = False): + """ + Verzameling plots van alle alpha's van de vreschillende parameters. + + :param geoprob_pipe: Hoofdopbject van de applicatie. + :param export: of de plot als html geëxporteerd moet worden, defaults to False + :return: plotly figuur + """ + self.geoprob_pipe = geoprob_pipe + self.export = export + self.model_string = get_geohydrological_model( + app_settings=self.geoprob_pipe.input_data.app_settings + ) + self.initial_input_mapper = INITIAL_INPUT_MAPPER[self.model_string][ + "input" + ] + + self._collect_data() + self.fig = self._create_plot() + self._export() + + def _collect_data(self): + """ + Verzamelen de data voor de alphas en metrering en voeg de distributies + en eenheden toe. + + :return: _description_ + """ + # Get data for graphing + df_alpha = self.geoprob_pipe.results.df_alphas_influence_factors_and_physical_values( + filter_deterministic=False, filter_derived=False + ) + df_alpha = df_alpha[ + [ + "uittredepunt_id", + "ondergrondscenario_id", + "vak_id", + "variable", + "distribution_type", + "physical_value", ] - )) - - # Layout and button - fig.update_layout( - height=300*len(parameters), - showlegend=False, - title=f"Overview of parameters for Scenario {scenario_orders[0]}", - updatemenus=[dict( - active=0, - buttons=buttons, - direction="down", - showactive=True, - x=1.05, y=1.01 - )], - ) - - # Export - if export: - export_dir = os.path.join( - geoprob_pipe.visualizations.graphs.export_dir, - "grafiek_physical_values" + ] + + gdf_uittredepunten = self.geoprob_pipe.input_data.uittredepunten.gdf + self.df_alpha = merge( + df_alpha, + gdf_uittredepunten[["uittredepunt_id", "metrering"]], + on="uittredepunt_id", + how="left", ) - os.makedirs(export_dir, exist_ok=True) - fig.write_html( - os.path.join(export_dir, "overview_alphas.html"), - include_plotlyjs='cdn' + # List of all alphas that can be shown + self.parameters: list[str] = list(df_alpha["variable"].unique()) + + # List of scenarios + self.scenarios = df_alpha["ondergrondscenario_id"].unique() + + # Add distribution type + self.dist_types = {} + for param in self.parameters: + dist_type = df_alpha.loc[ + df_alpha["variable"] == param, "distribution_type" + ].unique()[0] + self.dist_types.update({param: dist_type}) + + # Add units + unit_lookup = { + item["name"]: item["unit"] for item in self.initial_input_mapper + } + self.param_units = {} + + for param in self.parameters: + try: + self.param_units.update( + {param: str(unit_lookup[param]).strip("[]")} + ) + except KeyError: + self.param_units.update({param: "?"}) + + # Add parameter units missing in DUMMY_INPUT by hand. + self.param_units.update( + { + "L_kwelweg": "m", + "L_voorland": "m", + "W_voorland": "s/m", + "buitenwaterstand_gemiddeld": "m+NAP", + "d_deklaag": "m", + "dh_c": "m", + "dh_red": "m", + "dphi_c_u": "m+NAP", + "h_exit": "m+NAP", + "i_exit": "-", + "k_wvp": "m/dag", + "lambda_voorland": "m", + "phi_exit": "m+NAP", + "phi_exit_gemiddeld": "m+NAP", + "r_exit": "-", + "z_combin": "-", + "z_h": "-", + "z_p": "-", + "z_u": "-", + } ) - for scen_order in scenario_orders: - df_case = df[df["scenario_order"] == scen_order] - fig_case = make_subplots( - rows=len(parameters), cols=1, - shared_xaxes=False, - subplot_titles=parameters - ) - for row_idx, param in enumerate(parameters, start=1): + def _create_plot(self) -> Figure: + """ + Maak de subplots per paramate per scenario. Met maar een scenario zichtbaar. + Voegt bij de layout ook een knop toe om te kiezen welke set plots + zichtbaar zijn op basis van scenario naam. + + :return: Plotly figuur met de subplots. + """ + # Create subplots + fig = make_subplots( + rows=len(self.parameters), + cols=1, + shared_xaxes=False, + subplot_titles=self.parameters, + ) + + # Add a button for each ondergrondscenario + buttons = [] + + # Add scatter plot per scenario + for i, scenario in enumerate(self.scenarios): + df_case = self.df_alpha[ + self.df_alpha["ondergrondscenario_id"] == scenario + ] + + for row_idx, param in enumerate(self.parameters, start=1): df_param = df_case[df_case["variable"] == param] - fig_case.add_trace( + fig.add_trace( go.Scatter( x=df_param["metrering"], y=df_param["physical_value"], mode="markers", marker=dict(color="black", symbol="x", size=5), - name=param + name=param, + visible=(i == 0), ), - row=row_idx, col=1 + row=row_idx, + col=1, + ) + fig.update_xaxes( + showgrid=True, tickangle=90, row=row_idx, col=1 + ) + fig.update_yaxes( + showgrid=True, + title_text=f"{param} [{self.param_units[param]}]" + + f"
({self.dist_types[param]})", + row=row_idx, + col=1, + ) + + total_traces = len(self.scenarios) * len(self.parameters) + + # Determine which scenario is visible + vis = [False] * total_traces + vis[i * len(self.parameters) : (i + 1) * len(self.parameters)] = [ + True + ] * len(self.parameters) + + buttons.append( + dict( + label=f"{scenario}", + method="update", + args=[ + {"visible": vis}, + { + "title": f"Overview of parameters for Scenario {scenario}" + }, + ], ) - fig_case.update_xaxes(showgrid=True, tickangle=90, - row=row_idx, col=1) - fig_case.update_yaxes(showgrid=True, - title_text=f"{param} [{param_units[param]}]" - + f"
({dist_types[param]})", - row=row_idx, col=1) - - fig_case.update_layout( - height=300*len(parameters), - showlegend=False, - title=f"Overview of parameters for Scenario {scen_order}" ) - return fig + # Layout and button + fig.update_layout( + height=300 * len(self.parameters), + showlegend=False, + title=f"Overview of parameters for Scenario {self.scenarios[0]}", + updatemenus=[ + dict( + active=0, + buttons=buttons, + direction="down", + showactive=True, + x=1.05, + y=1.01, + ) + ], + ) + return fig + + def _export(self): + if self.export: + export_dir = os.path.join( + self.geoprob_pipe.visualizations.graphs.export_dir, + "grafiek_physical_values", + ) + + os.makedirs(export_dir, exist_ok=True) + + self.fig.write_html( + os.path.join(export_dir, "overview_alphas.html"), + include_plotlyjs="cdn", + ) diff --git a/geoprob_pipe/visualizations/graphs/river_waterlevel.py b/geoprob_pipe/visualizations/graphs/river_waterlevel.py index 51586f3c..dc5bb6d9 100644 --- a/geoprob_pipe/visualizations/graphs/river_waterlevel.py +++ b/geoprob_pipe/visualizations/graphs/river_waterlevel.py @@ -1,30 +1,42 @@ from __future__ import annotations + import os +from typing import TYPE_CHECKING, Dict, List, Tuple + import numpy as np -from pandas import DataFrame, concat, Series -from geopandas import GeoDataFrame import plotly.colors as pc +from geopandas import GeoDataFrame +from pandas import DataFrame, Series, concat from plotly.graph_objects import Figure, Scatter -from typing import TYPE_CHECKING, Dict, List, Tuple -from geoprob_pipe.cmd_app.parameter_input.expand_input_tables import run_expand_input_tables + +from geoprob_pipe.cmd_app.parameter_input.expand_input_tables import ( + run_expand_input_tables, +) + if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe - +# Terugkeertijden om buitenwaterstand bij te plotten TARGET_FREQS = np.array([ 0.1, 0.033333333, 0.01, 0.003333333, 0.001, 0.000333333, 0.0001, 3.33333E-05, 0.00001, 3.33333E-06]) def _collect_data(geoprob_pipe: GeoProbPipe) -> Tuple[DataFrame, GeoDataFrame, DataFrame, DataFrame]: + """ + Verzamel de data voor de plots. Physical values, Metrering, Beta waardes + en Hydra curves per uittredepunt. + :param geoprob_pipe: Hoofdbestand van de applicatie. + :return: Tuple met dataframes van alle onderdelen. + """ # Collect uittredepunten gdf_uittredepunten = geoprob_pipe.input_data.uittredepunten.gdf # Collect physical values - df = geoprob_pipe.results.df_alphas_influence_factors_and_physical_values( + df_physical = geoprob_pipe.results.df_alphas_influence_factors_and_physical_values( filter_deterministic=False, filter_derived=False) - df = df[["uittredepunt_id", "ondergrondscenario_id", "vak_id", "variable", "distribution_type", "physical_value"]] - df = df.merge(gdf_uittredepunten[["uittredepunt_id", "metrering"]], on="uittredepunt_id", how="left") + df_physical = df_physical[["uittredepunt_id", "ondergrondscenario_id", "vak_id", "variable", "distribution_type", "physical_value"]] + df_physical = df_physical.merge(gdf_uittredepunten[["uittredepunt_id", "metrering"]], on="uittredepunt_id", how="left") # Collect Beta results uittredepunten df_beta = geoprob_pipe.results.df_beta_uittredepunten @@ -38,14 +50,21 @@ def _collect_data(geoprob_pipe: GeoProbPipe) -> Tuple[DataFrame, GeoDataFrame, D df_input['parameter_input'].apply(Series)], # Expansion of dict in parameter_input-column axis=1) - return df, gdf_uittredepunten, df_beta, df_input + return df_physical, gdf_uittredepunten, df_beta, df_input def _collect_hydra_curves( - df: DataFrame, gdf_uittredepunten: GeoDataFrame) -> Dict[float, Dict[str, List[float]]]: + df_input: DataFrame, gdf_uittredepunten: GeoDataFrame) -> Dict[float, Dict[str, List[float]]]: + """ + Bepaal de hydra curves op de juiste locatie. + + :param df_input: DataFreame met input tabellen voor hrd locaties. + :param gdf_uittredepunten: GeoDataFrame met metrering. + :return: Hydra curves per locatie als dict. + """ hydra_curves = {freq: {"metrering": [], "level": []} for freq in TARGET_FREQS} - for _, row in df.iterrows(): + for _, row in df_input.iterrows(): if row["distribution_type"] != "deterministic": df_subset = gdf_uittredepunten[gdf_uittredepunten["vak_id"] == row["vak_id"]] if df_subset.empty: @@ -74,8 +93,16 @@ def _collect_hydra_curves( def _plot_continuous_exceedance_lines( fig: Figure, hydra_curves: Dict[float, Dict[str, List[float]]]): + """ + Plot de hydra curves als continue lijnen per herhaaltijd in verschillende + kleuren. - # Blue gradient for lines + :param fig: plotly.Figure object met de plots. + :param hydra_curves: Dict met Hydra curves per locatie + :return: fig + """ + + # Multicolor gradient for lines line_colors = pc.sample_colorscale( colorscale="Jet", samplepoints=np.linspace(start=0.2, stop=0.9, num=len(TARGET_FREQS))) freq_color_map = {f: c for f, c in zip(TARGET_FREQS, line_colors)} @@ -95,6 +122,14 @@ def _plot_continuous_exceedance_lines( def _plot_physical_values(fig: Figure, df: DataFrame) -> Figure: + """ + Plot per uitredepunt de physical value van de buitenwaterstand met een kleur + op basis van de beta waarde. + + :param fig: plotly.Figure object met de plots. + :param df: Dataframe met de betas. + :return: plotly.Figure object met de plots. + """ beta_colorscale = "RdYlGn" # Green-to-red scale (low beta = red, high beta = green) betas = df["beta"].to_numpy() fig.add_trace( @@ -129,13 +164,21 @@ def _update_layout(fig: Figure) -> Figure: def river_waterlevel(geoprob_pipe: GeoProbPipe, export: bool = False): + """ + Maak een plot van de hydra curves voor de buitenwaterstand bij verschillende + herhaaltijden samen met de physical value die berekent zijn. + + :param geoprob_pipe: Hoofdbestand van de applicatie. + :param export: of het figuur naar html moet worden geëxporteerd, defaults to False + :return: plotly.Figure object + """ # Prepare base data df, gdf_uittredepunten, df_beta, df_input = _collect_data(geoprob_pipe=geoprob_pipe) # Collect Hydra lines (grouped per frequency) hydra_curves: Dict[float, Dict[str, List[float]]] = _collect_hydra_curves( - df=df_input, gdf_uittredepunten=gdf_uittredepunten) + df_input=df_input, gdf_uittredepunten=gdf_uittredepunten) # Plot one continuous line per exceedance frequency fig = Figure() diff --git a/geoprob_pipe/visualizations/maps/betamap.py b/geoprob_pipe/visualizations/maps/betamap.py index f5bd9291..aa21210c 100644 --- a/geoprob_pipe/visualizations/maps/betamap.py +++ b/geoprob_pipe/visualizations/maps/betamap.py @@ -1,66 +1,85 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Tuple -import plotly.graph_objects as go + import os +from typing import TYPE_CHECKING, Dict, List, Tuple + import geopandas as gpd -from shapely.geometry import LineString, MultiLineString, GeometryCollection +import plotly.graph_objects as go +import fiona +from shapely.geometry import GeometryCollection, LineString, MultiLineString if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe -def _add_line(geoprob_pipe: GeoProbPipe, fig: go.Figure, - layer: str, color: str): - """ Helperfunctie om de lijnen uit de geopackage te vinden en toe te voegen aan de map. Layer is de naam van de - laag in de geopackage waar de lijn is opgeslagen. Color is de kleur van deze lijn in de map. """ - - gdf_traject = gpd.read_file( - geoprob_pipe.input_data.app_settings.geopackage_filepath, - layer=layer) - gdf_traject = gdf_traject.to_crs("EPSG:4326") - - def plot_linestring(ls, display): - xs, ys = ls.xy - xs = list(xs) - ys = list(ys) - fig.add_trace(go.Scattermap( - lon=xs, - lat=ys, - mode="lines", - line=dict(color=color, width=1), - hoverinfo="none", - name=layer, - legendgroup=layer, - showlegend=display - )) - - show = True - for geom in gdf_traject.geometry: - if isinstance(geom, LineString): - plot_linestring(geom, show) - show = False - - elif isinstance(geom, MultiLineString): - for line in geom.geoms: - plot_linestring(line, show) - show = False - - elif isinstance(geom, GeometryCollection): - for g in geom.geoms: - if isinstance(g, LineString): - plot_linestring(g, show) - show = False - elif isinstance(g, MultiLineString): - for line in g.geoms: - plot_linestring(line, show) - show = False - - else: - print("Skipping unsupported geometry:", geom.geom_type) +def _add_line( + geoprob_pipe: GeoProbPipe, fig: go.Figure, layer: str, color: str +): + """Helperfunctie om de lijnen uit de geopackage te vinden en toe te voegen aan de map. Layer is de naam van de + laag in de geopackage waar de lijn is opgeslagen. Color is de kleur van deze lijn in de map.""" + + # Check of er maar een intredelijn of dat er meerdere zijn ingevoerd. + gpkg_layers: list[str] = fiona.listlayers( + geoprob_pipe.input_data.app_settings.geopackage_filepath + ) + list_layers: list[str] = [ + lay for lay in gpkg_layers if lay.split("__")[0] == layer + ] + + for listed_layer in list_layers: + gdf_traject = gpd.read_file( + geoprob_pipe.input_data.app_settings.geopackage_filepath, + layer=listed_layer, + ) + gdf_traject = gdf_traject.to_crs("EPSG:4326") + + def plot_linestring(ls, display): + xs, ys = ls.xy + xs = list(xs) + ys = list(ys) + fig.add_trace( + go.Scattermap( + lon=xs, + lat=ys, + mode="lines", + line=dict(color=color, width=1), + hoverinfo="none", + name=listed_layer, + legendgroup=layer, + showlegend=display, + ) + ) + + display = True + for geom in gdf_traject.geometry: + if isinstance(geom, LineString): + plot_linestring(geom, display) + display = False + + elif isinstance(geom, MultiLineString): + for line in geom.geoms: + plot_linestring(line, display) + display = False + + elif isinstance(geom, GeometryCollection): + for g in geom.geoms: + if isinstance(g, LineString): + plot_linestring(g, display) + display = False + elif isinstance(g, MultiLineString): + for line in g.geoms: + plot_linestring(line, display) + display = False + + else: + print("Skipping unsupported geometry:", geom.geom_type) return fig -def _generate_colorscale(cg: Dict[str, List], labels: List[str]) -> List[Tuple[float, str]]: + +def _generate_colorscale( + cg: Dict[str, List], labels: List[str] +) -> List[Tuple[float, str]]: # cg > category label > [van beta, tot beta] upper_boundary_beta_graph = cg[labels[0]][1] lower_boundary_beta_graph = cg[labels[-1]][0] @@ -70,14 +89,23 @@ def _generate_colorscale(cg: Dict[str, List], labels: List[str]) -> List[Tuple[f colorscale_intervals: Dict = {} for index, label in enumerate(labels): cg_beta_bovengrens = cg[label][1] - colorscale_intervals[index] = (cg_beta_bovengrens - lower_boundary_beta_graph) / beta_range_graph + colorscale_intervals[index] = ( + cg_beta_bovengrens - lower_boundary_beta_graph + ) / beta_range_graph # Other end boundary value: 0.0 colorscale_intervals[labels.__len__()] = 0.0 # Color scale - colors = ["rgb(30,141,41)", "rgb(146,206,90)", "rgb(198,226,176)", "rgb(255,255,0)", "rgb(254,165,3)", - "rgb(255,0,0)", "rgb(177,33,38)"] + colors = [ + "rgb(30,141,41)", + "rgb(146,206,90)", + "rgb(198,226,176)", + "rgb(255,255,0)", + "rgb(254,165,3)", + "rgb(255,0,0)", + "rgb(177,33,38)", + ] colorscale = [ (colorscale_intervals[7], colors[6]), (colorscale_intervals[6], colors[6]), @@ -99,7 +127,6 @@ def _generate_colorscale(cg: Dict[str, List], labels: List[str]) -> List[Tuple[f class BetaMap: - def __init__(self, geoprob_pipe: GeoProbPipe, export: bool = False): self.geoprob_pipe = geoprob_pipe @@ -117,24 +144,33 @@ def _import_results(self): # results import self.inp_point = self.geoprob_pipe.input_data.uittredepunten.gdf self.res_sc = self.geoprob_pipe.results.df_beta_scenarios_final - mask = self.inp_point["uittredepunt_id"].isin(self.res_sc["uittredepunt_id"]) + mask = self.inp_point["uittredepunt_id"].isin( + self.res_sc["uittredepunt_id"] + ) self.inp_point = self.inp_point[mask] # Setup of beta category limits - self.cg: Dict[str, List] = self.geoprob_pipe.input_data.traject_normering.riskeer_categorie_grenzen + self.cg: Dict[str, List] = ( + self.geoprob_pipe.input_data.traject_normering.riskeer_categorie_grenzen + ) self.labels: List[str] = list(self.cg.keys()) def _setup_gdf(self): self.hoverdata = ["uittredepunt_id", "converged", "beta"] - self.df = self.res_sc.merge(self.inp_point, on="uittredepunt_id", how="inner") + self.df = self.res_sc.merge( + self.inp_point, on="uittredepunt_id", how="inner" + ) idx = self.df.groupby(["uittredepunt_id"])["beta"].idxmin() self.df = self.df.loc[idx] self.gdf = gpd.GeoDataFrame( self.df, - geometry=gpd.points_from_xy(self.inp_point.geometry.x, self.inp_point.geometry.y), - crs="EPSG:28992") + geometry=gpd.points_from_xy( + self.inp_point.geometry.x, self.inp_point.geometry.y + ), + crs="EPSG:28992", + ) # Transformeer naar WGS84 (latitude / longitude) self.gdf_latlon = self.gdf.to_crs("EPSG:4326") @@ -167,53 +203,65 @@ def _calculate_zoom(lat_range, lon_range): def _create_figure(self): self.fig = go.Figure() - self.fig.add_trace(go.Scattermap( - mode="markers", - lat=self.gdf_latlon.geometry.y, - lon=self.gdf_latlon.geometry.x, - marker=dict(size=9, color="black"), - showlegend=False)) - self.fig.add_trace(go.Scattermap( - mode='markers', - lat=self.gdf_latlon.geometry.y, # direct uit geometrie - lon=self.gdf_latlon.geometry.x, # direct uit geometrie - marker=dict( - size=8, - color=self.gdf_latlon['beta'], - colorscale=_generate_colorscale(cg=self.cg, labels=self.labels), - cmin=self.cg[self.labels[6]][0], - cmax=self.cg[self.labels[0]][1], - colorbar=dict( - title="Bèta, WBI cat.", - tickvals=[ - self.cg[self.labels[6]][0], - self.cg[self.labels[6]][1], - self.cg[self.labels[5]][1], - self.cg[self.labels[4]][1], - self.cg[self.labels[3]][1], - self.cg[self.labels[2]][1], - self.cg[self.labels[1]][1], - self.cg[self.labels[0]][1] - ], - ticktext=[f"{v:.2f}" for v in [ - self.cg[self.labels[6]][0], - self.cg[self.labels[6]][1], - self.cg[self.labels[5]][1], - self.cg[self.labels[4]][1], - self.cg[self.labels[3]][1], - self.cg[self.labels[2]][1], - self.cg[self.labels[1]][1], - self.cg[self.labels[0]][1]]], - ) + self.fig.add_trace( + go.Scattermap( + mode="markers", + lat=self.gdf_latlon.geometry.y, + lon=self.gdf_latlon.geometry.x, + marker=dict(size=9, color="black"), + showlegend=False, + ) + ) + self.fig.add_trace( + go.Scattermap( + mode="markers", + lat=self.gdf_latlon.geometry.y, # direct uit geometrie + lon=self.gdf_latlon.geometry.x, # direct uit geometrie + marker=dict( + size=8, + color=self.gdf_latlon["beta"], + colorscale=_generate_colorscale( + cg=self.cg, labels=self.labels + ), + cmin=self.cg[self.labels[6]][0], + cmax=self.cg[self.labels[0]][1], + colorbar=dict( + title="Bèta, WBI cat.", + tickvals=[ + self.cg[self.labels[6]][0], + self.cg[self.labels[6]][1], + self.cg[self.labels[5]][1], + self.cg[self.labels[4]][1], + self.cg[self.labels[3]][1], + self.cg[self.labels[2]][1], + self.cg[self.labels[1]][1], + self.cg[self.labels[0]][1], + ], + ticktext=[ + f"{v:.2f}" + for v in [ + self.cg[self.labels[6]][0], + self.cg[self.labels[6]][1], + self.cg[self.labels[5]][1], + self.cg[self.labels[4]][1], + self.cg[self.labels[3]][1], + self.cg[self.labels[2]][1], + self.cg[self.labels[1]][1], + self.cg[self.labels[0]][1], + ] + ], + ), ), - hoverinfo='text', - text=self.gdf_latlon[self.hoverdata].apply( - lambda row: '
'.join( - [f"{col}: {row[col]}" for col in self.hoverdata] + hoverinfo="text", + text=self.gdf_latlon[self.hoverdata].apply( + lambda row: "
".join( + [f"{col}: {row[col]}" for col in self.hoverdata] ), - axis=1), - showlegend=False - )) + axis=1, + ), + showlegend=False, + ) + ) # Layout self.fig.update_layout( @@ -222,33 +270,35 @@ def _create_figure(self): map_zoom=self.zoom, map_center=dict( lat=self.gdf_latlon.geometry.y.mean(), - lon=self.gdf_latlon.geometry.x.mean() + lon=self.gdf_latlon.geometry.x.mean(), ), legend=dict( - orientation="h", - yanchor="bottom", - y=1.02, - xanchor="right", - x=1 + orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1 ), - dragmode='zoom', - title='Faalkansberekening STPH' + dragmode="zoom", + title="Faalkansberekening STPH", ) def _add_lines(self): - self.fig = _add_line(self.geoprob_pipe, self.fig, - "dijktraject", "black") - self.fig = _add_line(self.geoprob_pipe, self.fig, - "intredelijn", "blue") - self.fig = _add_line(self.geoprob_pipe, self.fig, - "binnenteenlijn", "purple") - self.fig = _add_line(self.geoprob_pipe, self.fig, - "buitenteenlijn", "red") + self.fig = _add_line( + self.geoprob_pipe, self.fig, "dijktraject", "black" + ) + self.fig = _add_line( + self.geoprob_pipe, self.fig, "intredelijn", "blue" + ) + self.fig = _add_line( + self.geoprob_pipe, self.fig, "binnenteenlijn", "purple" + ) + self.fig = _add_line( + self.geoprob_pipe, self.fig, "buitenteenlijn", "red" + ) def _optionally_export(self): if self.export: path = self.geoprob_pipe.visualizations.maps.export_dir - self.fig.write_html(os.path.join( - path, 'Faalkansberekening STPH.html'), include_plotlyjs='cdn') + self.fig.write_html( + os.path.join(path, "Faalkansberekening STPH.html"), + include_plotlyjs="cdn", + ) # self.fig.write_image(os.path.join( # path, 'Faalkansberekening STPH.png'), format='png') diff --git a/tests/cmd_app/parameter_input/test_expand_input_tables.py b/tests/cmd_app/parameter_input/test_expand_input_tables.py new file mode 100644 index 00000000..63eae4e1 --- /dev/null +++ b/tests/cmd_app/parameter_input/test_expand_input_tables.py @@ -0,0 +1,231 @@ +""" +Voor idere rij in de excel en gis moet worden gecheckt of deze op de juiste +plaats in de expanded staat. En of er geen hogere prioriteit aanwezig is. +traject->vak->utp +scenario->wildcard +""" + +import ast +import sqlite3 + +import numpy as np +import pandas as pd + +# TODO Permanete test gpkg als deze naar de nieuwe standaard is gezet +gpkg_filepath = r"C:\Users\vinji\Python\Libraries\GEOprob-Pipe\Workspace\fix.geoprob_pipe.gpkg" + + +def get_mean(x): + if isinstance(x, dict): + return x.get("mean") + elif isinstance(x, str): + d = ast.literal_eval(x) # string → dict + return d.get("mean", np.nan) + return None + + +def _check_match(df_check: pd.DataFrame, df_compare: pd.DataFrame, key: str): + rel_tol = 1e-4 + col = key.replace("mean", "match") + mask_nan = df_compare[key].isna() + + df_check[col] = np.isclose( + df_compare["mean"], df_compare[key], rtol=rel_tol + ) + + df_check[col] = df_check[col].mask(mask_nan, None) + + return df_check + + +def validate_expand_tables(): + # --- Verzamel data op uit database + conn = sqlite3.connect(gpkg_filepath) + df_out = pd.read_sql("SELECT * FROM data__expanded_parameters", conn) + df_excel = pd.read_sql("SELECT * FROM data__excel_parameter_invoer", conn) + df_excel["scope_referentie"] = df_excel["scope_referentie"].astype("Int64") + df_gis = pd.read_sql("SELECT * FROM data__gis_parameter_invoer", conn) + df_gis["scope_referentie"] = df_gis["scope_referentie"].astype("Int64") + df_gis["mean"] = df_gis["mean"].astype(float) + conn.close() + + df_gis["scope"] = "gis_uittredepunt" + # Verwijder lege kolommen + df_excel = df_excel.dropna(axis=1, how="all") + df_gis = df_gis.dropna(axis=1, how="all") + df_input = pd.concat([df_excel, df_gis], ignore_index=True) + + # --- Clean dataframes + df_input["ondergrondscenario_naam"] = df_input[ + "ondergrondscenario_naam" + ].replace("", pd.NA) + df_out["ondergrondscenario_naam"] = df_out[ + "ondergrondscenario_naam" + ].replace("", pd.NA) + df_out["uittredepunt_id"] = df_out["uittredepunt_id"].astype("Int64") + df_out = df_out.rename(columns={"parameter_name": "parameter"}) + + # Filter Dataframes + df_out["mean"] = df_out["parameter_input"].apply(get_mean) + df_out = df_out[ + [ + "parameter", + "vak_id", + "uittredepunt_id", + "ondergrondscenario_naam", + "mean", + ] + ] + df_input = df_input[ + [ + "parameter", + "scope", + "scope_referentie", + "ondergrondscenario_naam", + "mean", + ] + ] + + # --- Merge per stap + df_compare = df_out.copy() + + # Traject + df_traject = df_input.loc[df_input["scope"] == "traject"] + df_compare = df_compare.merge( + df_traject[["parameter", "mean"]], + on=["parameter"], + how="left", + suffixes=["", "_traject"], + ) + + # Vak zonder scenario (Wildcard) + df_vak_wild = df_input.loc[ + (df_input["scope"] == "vak") + & (df_input["ondergrondscenario_naam"].isna()) + ] + df_vak_wild = df_vak_wild.rename(columns={"scope_referentie": "vak_id"}) + df_compare = df_compare.merge( + df_vak_wild[["parameter", "vak_id", "mean"]], + on=["parameter", "vak_id"], + how="left", + suffixes=["", "_vak_wild"], + ) + + # Vak met scenario + df_vak_scen = df_input.loc[ + (df_input["scope"] == "vak") + & (df_input["ondergrondscenario_naam"].notna()) + ] + df_vak_scen = df_vak_scen.rename(columns={"scope_referentie": "vak_id"}) + df_compare = df_compare.merge( + df_vak_scen[ + ["parameter", "vak_id", "ondergrondscenario_naam", "mean"] + ], + on=["parameter", "vak_id", "ondergrondscenario_naam"], + how="left", + suffixes=["", "_vak_scen"], + ) + + # gis_utp zonder scenario (Wildcard) + df_gis_utp_wild = df_input.loc[ + (df_input["scope"] == "gis_uittredepunt") + & (df_input["ondergrondscenario_naam"].isna()) + ] + df_gis_utp_wild = df_gis_utp_wild.rename( + columns={"scope_referentie": "uittredepunt_id"} + ) + df_compare = df_compare.merge( + df_gis_utp_wild[["parameter", "uittredepunt_id", "mean"]], + on=["parameter", "uittredepunt_id"], + how="left", + suffixes=["", "_gis_utp_wild"], + ) + + # excel_upt zonder scenario (Wildcard) + df_excel_utp_wild = df_input.loc[ + (df_input["scope"] == "uittredepunt") + & (df_input["ondergrondscenario_naam"].isna()) + ] + df_excel_utp_wild = df_excel_utp_wild.rename( + columns={"scope_referentie": "uittredepunt_id"} + ) + df_compare = df_compare.merge( + df_excel_utp_wild[["parameter", "uittredepunt_id", "mean"]], + on=["parameter", "uittredepunt_id"], + how="left", + suffixes=["", "_excel_utp_wild"], + ) + # gis_utp met scenario + df_gis_utp_scen = df_input.loc[ + (df_input["scope"] == "gis_uittredepunt") + & (df_input["ondergrondscenario_naam"].notna()) + ] + df_gis_utp_scen = df_gis_utp_scen.rename( + columns={"scope_referentie": "uittredepunt_id"} + ) + df_compare = df_compare.merge( + df_gis_utp_scen[ + ["parameter", "uittredepunt_id", "ondergrondscenario_naam", "mean"] + ], + on=["parameter", "uittredepunt_id", "ondergrondscenario_naam"], + how="left", + suffixes=["", "_gis_utp_scen"], + ) + # excel_utp met scenario + df_excel_utp_scen = df_input.loc[ + (df_input["scope"] == "uittredepunt") + & (df_input["ondergrondscenario_naam"].notna()) + ] + df_excel_utp_scen = df_excel_utp_wild.rename( + columns={"scope_referentie": "uittredepunt_id"} + ) + df_compare = df_compare.merge( + df_excel_utp_scen[["parameter", "uittredepunt_id", "mean"]], + on=["parameter", "uittredepunt_id"], + how="left", + suffixes=["", "_excel_utp_scen"], + ) + + # --- Check of waardes overeenkomen + + df_check = df_out.copy() + + df_check = _check_match(df_check, df_compare, "mean_traject") + df_check = _check_match(df_check, df_compare, "mean_vak_wild") + df_check = _check_match(df_check, df_compare, "mean_vak_scen") + df_check = _check_match(df_check, df_compare, "mean_gis_utp_wild") + df_check = _check_match(df_check, df_compare, "mean_excel_utp_wild") + df_check = _check_match(df_check, df_compare, "mean_gis_utp_scen") + df_check = _check_match(df_check, df_compare, "mean_excel_utp_scen") + + df_errors = df_out.copy() + cols = [ + "match_traject", + "match_vak_wild", + "match_vak_scen", + "match_gis_utp_wild", + "match_excel_utp_wild", + "match_gis_utp_scen", + "match_excel_utp_scen", + ] + df_errors["match"] = df_check[cols].any(axis=1) + + # Hoogste prioriteit check + df_errors["highest_priority"] = ( + df_check[cols].idxmax(axis=1).where(df_errors["match"], None) + ) + + arr = df_check[cols].to_numpy(dtype=object) + + df_errors["priority_error"] = [ + any(v is False for v in row[(max([i for i, x in enumerate(row) if x is True], default=-1) + 1):]) + for row in arr + ] + + # Result + print(df_errors.loc[ + (~df_errors["match"]) | (df_errors["priority_error"]) + ]) + + +validate_expand_tables() diff --git a/tests/results/assemblage/test_assemblage.py b/tests/results/assemblage/test_assemblage.py index 976378b1..102c3a79 100644 --- a/tests/results/assemblage/test_assemblage.py +++ b/tests/results/assemblage/test_assemblage.py @@ -31,9 +31,10 @@ def test_window(): window_size=10, point_list=list_dsn, m_van=0, m_tot=40 ) + print(elements) assert sum_pf == pytest.approx(1.78e-12) assert max_pf == pytest.approx(1e-12) - assert elements.__len__() == 4 + assert len(elements) == 4 assert elements[0].kans_dsn.pf == 0.0