From 80b9f4ae4f550a2feb64a0c23ad8e5a0abf0ccf8 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 14 Aug 2026 12:19:45 +0200 Subject: [PATCH 1/5] Assertion that length uittredepunten remains the same --- geoprob_pipe/changelog.py | 3 +++ geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py | 5 ++++- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/geoprob_pipe/changelog.py b/geoprob_pipe/changelog.py index e8b8c14b..7e6d0560 100644 --- a/geoprob_pipe/changelog.py +++ b/geoprob_pipe/changelog.py @@ -1,5 +1,8 @@ CHANGELOG = { + "2.2.12": + "Validatie op dubbele punten bij het inlezen van de HRD-punten. Oorzaak is database 21-2 waar een dubbeling in " + "zit. Direct controle op dubbeling in uittredepunten toegevoegd. ", "2.2.11": "Bug fix. When there were NaN-calculation results (PoF), it resulted in a crash in the KansElement.", "2.2.10": diff --git a/geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py b/geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py index 371f2d16..f4182664 100644 --- a/geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py +++ b/geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py @@ -30,8 +30,11 @@ def coupled_hrd_to_uittredepunten(app_settings: ApplicationSettings) -> bool: return True # Assuming already added # Perform spatial join to find the nearest HRD-location for each Exit Point - gdf_exit_with_hrd = gdf_exit_points.sjoin_nearest( + gdf_exit_with_hrd: GeoDataFrame = gdf_exit_points.sjoin_nearest( gdf_hrd_locations[['geometry', 'location_name']], how='left', distance_col='distance') + assert gdf_exit_points.__len__() == gdf_exit_with_hrd.__len__(), \ + (f"While coupling the HRD-location to the uittredepunten something went wrong. Please contact a developer with " + f"the traceback.") # Define which columns to keep (after spatial join) columns_to_keep = list(gdf_exit_points.columns) diff --git a/pyproject.toml b/pyproject.toml index 74a611ef..504fb7bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "geoprob_pipe" -version = "2.2.11" +version = "2.2.12" description = "Execute probabilistic piping calculations using the Deltares probabilic library." keywords = [ "levee", From 4863213fdce585b3af5045fce99c18cb43cc60e6 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 14 Aug 2026 14:49:14 +0200 Subject: [PATCH 2/5] Validatie HRD-locatie toegevoegd --- .../spatial_layers/hrd/import_from_hrd.py | 40 +++++++++++++++++-- geoprob_pipe/utils/gdf.py | 14 +++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py index 59f5450e..8d424629 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py +++ b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py @@ -6,6 +6,8 @@ from geopandas import GeoDataFrame, read_file from shapely import Point import os + +from geoprob_pipe.utils.gdf import validate_unique_point_locations from geoprob_pipe.utils.validation_messages import BColors import warnings from geoprob_pipe.cmd_app.spatial_layers.hrd.utils import hrd_file_path @@ -15,6 +17,7 @@ from pandas import DataFrame, concat from typing import Optional, List, Tuple import sqlite3 +import sys if TYPE_CHECKING: from geoprob_pipe.cmd_app.cmd import ApplicationSettings @@ -65,9 +68,9 @@ def _ask_path_to_hrd_dir() -> str: return filepath -def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: str): +def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: str) -> bool: - # Add HRD locations to GeoPackage + # Collect HRD-locations hrd_path = hrd_file_path(hrd_dir=hrd_dir) hrd = pydra.HRDatabase(hrd_path) location_names = hrd.locationnames @@ -87,10 +90,34 @@ def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: s "location_name": location_name, "geometry": Point(hrd_location.settings.x_coordinate, hrd_location.settings.y_coordinate) }) - gdf = GeoDataFrame(hrd_location_rows, columns=['location_name', 'geometry'], crs='EPSG:28992') + # TODO Runtime improvement: the below method of Pydra-Core, to only collect the Hydra-Locations, is relatively slow. + # We could just read the sqlite HRD-database directly and store the HRD-locations. + + # Validate all points are unique + success, failure_msg = validate_unique_point_locations(gdf=gdf, id_column="location_name") + if not success: + print(f"{BColors.WARNING}" + f"Er zijn één of meerdere dubbelingen in de HRD-locaties. " + f"Uit de validatie volgt het volgende bericht: " + f"{failure_msg} " + f"Corrigeer de HRD-database, of vul de overschrijdingsfrequentielijnen handmatig in. " + f"{BColors.ENDC}") + + # Verwijder importeren van HRD als keuze + file_path = app_settings.geopackage_filepath + conn = sqlite3.connect(file_path) + cursor = conn.cursor() + cursor.execute(f"DELETE FROM geoprob_pipe_metadata WHERE metadata_type = 'extract_hrd_data';") + conn.commit() + cursor.close() + + return False + + # Add HRD locations to GeoPackage gdf.to_file(Path(app_settings.geopackage_filepath), layer="hrd_locaties", driver="GPKG") print(BColors.OKBLUE, f"✅ HRD-locatie punten toegevoegd aan GeoProb-Pipe GeoPackage.", BColors.ENDC) + return True def _add_hrd_overschrijdingsfrequentielijnen(hrd_dir: str, app_settings: ApplicationSettings): @@ -219,6 +246,11 @@ def _add_traject_parameters(app_settings: ApplicationSettings, hrd_dir: str): def import_from_hrd(app_settings: ApplicationSettings): hrd_dir = _ask_path_to_hrd_dir() - _add_hrd_locations_to_database(app_settings=app_settings, hrd_dir=hrd_dir) + + # Import HRD-locations + if not _add_hrd_locations_to_database(app_settings=app_settings, hrd_dir=hrd_dir): + sys.exit("Applicatie afgesloten") + + # Other actions _add_hrd_overschrijdingsfrequentielijnen(hrd_dir=hrd_dir, app_settings=app_settings) _add_traject_parameters(app_settings=app_settings, hrd_dir=hrd_dir) diff --git a/geoprob_pipe/utils/gdf.py b/geoprob_pipe/utils/gdf.py index 0cad7a26..3fb4e917 100644 --- a/geoprob_pipe/utils/gdf.py +++ b/geoprob_pipe/utils/gdf.py @@ -55,3 +55,17 @@ def validate_vakindeling_merges_to_single_linestring(gdf: GeoDataFrame) -> Tuple """ Controleert of alle LineStrings aaneengesloten zijn. """ merged = line_merge(unary_union(gdf.geometry)) return isinstance(merged, LineString), type(merged) + + +def validate_unique_point_locations(gdf: GeoDataFrame, id_column: str) -> Tuple[bool, str]: + """ Validates if the given GeoDataFrame has only unique point-locations. If not, it returns a failure message + with in there the non-unique ids. """ + + filter_duplicate_geometries = gdf.geometry.duplicated(keep=False) + gdf_duplicates = gdf[filter_duplicate_geometries] + + if gdf_duplicates.__len__() == 0: + return True, "" + + non_unique_ids = gdf_duplicates[id_column].values.tolist() + return False, f"Identifiers {non_unique_ids} have non-unique geometries." From 188a96136ae3715f095e93374dec8155b5045572 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 14 Aug 2026 16:22:17 +0200 Subject: [PATCH 3/5] Added validation on uittredepunt locaties --- .../spatial_layers/hrd/import_from_hrd.py | 3 +-- .../uittredepunten/uittredepunten.py | 12 +++++++++ geoprob_pipe/utils/gdf.py | 26 ++++++++++++++++--- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py index 8d424629..cee46ff4 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py +++ b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py @@ -99,8 +99,7 @@ def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: s if not success: print(f"{BColors.WARNING}" f"Er zijn één of meerdere dubbelingen in de HRD-locaties. " - f"Uit de validatie volgt het volgende bericht: " - f"{failure_msg} " + f"Uit de validatie volgt het volgende bericht: {failure_msg} " f"Corrigeer de HRD-database, of vul de overschrijdingsfrequentielijnen handmatig in. " f"{BColors.ENDC}") diff --git a/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py b/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py index 2d7a2733..e1149db0 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py +++ b/geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py @@ -7,6 +7,8 @@ from shapely import Point from geopandas import GeoDataFrame, read_file import fiona + +from geoprob_pipe.utils.gdf import validate_unique_point_locations from geoprob_pipe.utils.validation_messages import BColors import warnings if TYPE_CHECKING: @@ -189,6 +191,16 @@ def request_uittredepunten_filepath(app_settings: ApplicationSettings): f"andere typen geometrie. Enkel punten zijn toegestaan.", BColors.ENDC) request_uittredepunten_filepath(app_settings=app_settings) + # Confirm all are unique locations + succes, failure_msg = validate_unique_point_locations(gdf=gdf, return_coords=True) + if not succes: + print(f"{BColors.WARNING}" + f"Er zijn één of meerdere dubbelingen in de uittredepunten-locaties. " + f"Uit de validatie volgt het volgende bericht: {failure_msg} " + f"Corrigeer het bestand en importeer opnieuw de uittredepunten. " + f"{BColors.ENDC}") + request_uittredepunten_filepath(app_settings=app_settings) + # Continue questionnaire specify_column_with_maaiveld_niveau(app_settings, gdf=gdf) diff --git a/geoprob_pipe/utils/gdf.py b/geoprob_pipe/utils/gdf.py index 3fb4e917..b4de1415 100644 --- a/geoprob_pipe/utils/gdf.py +++ b/geoprob_pipe/utils/gdf.py @@ -57,9 +57,16 @@ def validate_vakindeling_merges_to_single_linestring(gdf: GeoDataFrame) -> Tuple return isinstance(merged, LineString), type(merged) -def validate_unique_point_locations(gdf: GeoDataFrame, id_column: str) -> Tuple[bool, str]: +def validate_unique_point_locations( + gdf: GeoDataFrame, id_column: str | None = None, return_coords: bool = False) -> Tuple[bool, str]: """ Validates if the given GeoDataFrame has only unique point-locations. If not, it returns a failure message - with in there the non-unique ids. """ + with in there the non-unique ids. + + :param gdf: + :param id_column: Specifies which column identifies each point. If None, index will be used as identifiers. + :param return_coords: If True, the failure message will also include the RD-coordinates of the points. + :return: + """ filter_duplicate_geometries = gdf.geometry.duplicated(keep=False) gdf_duplicates = gdf[filter_duplicate_geometries] @@ -67,5 +74,16 @@ def validate_unique_point_locations(gdf: GeoDataFrame, id_column: str) -> Tuple[ if gdf_duplicates.__len__() == 0: return True, "" - non_unique_ids = gdf_duplicates[id_column].values.tolist() - return False, f"Identifiers {non_unique_ids} have non-unique geometries." + # Collect ids + ids_of_non_unique_pnts = gdf_duplicates.index.tolist() + if id_column is not None: + ids_of_non_unique_pnts = gdf_duplicates[id_column].values.tolist() + failure_msg = f"Identifiers {ids_of_non_unique_pnts} have non-unique geometries." + + # Collect coordinates + if return_coords: + coords = [f"{pnt.x}, {pnt.y}" for pnt in gdf_duplicates.geometry.values.tolist()] + id_coord_list = [f"{identifier} ({pnt})" for identifier, pnt in zip(ids_of_non_unique_pnts, coords)] + failure_msg = f"Identifiers {id_coord_list} have non-unique geometries." + + return False, failure_msg From 77887cd7b9a936e6e59fa2c78bd41eb5b545cc61 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Tue, 18 Aug 2026 10:16:29 +0200 Subject: [PATCH 4/5] Verduidelinking comment --- geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py index cee46ff4..a46011c4 100644 --- a/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py +++ b/geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py @@ -103,7 +103,7 @@ def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: s f"Corrigeer de HRD-database, of vul de overschrijdingsfrequentielijnen handmatig in. " f"{BColors.ENDC}") - # Verwijder importeren van HRD als keuze + # Verwijder eerder gemaakte keuze 'Importeren van HRD-database' (zodat je opnieuw de keuze kunt maken) file_path = app_settings.geopackage_filepath conn = sqlite3.connect(file_path) cursor = conn.cursor() From e93f4b5df72f9805c7cccd698503f73170e99715 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Tue, 18 Aug 2026 10:19:55 +0200 Subject: [PATCH 5/5] beta toegevoegd aan unit test --- .github/workflows/testing_pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing_pull_request.yml b/.github/workflows/testing_pull_request.yml index 1fbef999..959bab6c 100644 --- a/.github/workflows/testing_pull_request.yml +++ b/.github/workflows/testing_pull_request.yml @@ -3,7 +3,7 @@ name: Automated testing and coverage (during pull request) on: pull_request: - branches: [ "alpha" ] + branches: [ "beta", "alpha" ] paths: - 'geoprob_pipe/**' - 'test_system.py'