108 Dubbele uittredepunten door HRD-punten join

.github/workflows/testing_pull_request.yml

@@ -3,7 +3,7 @@ name: Automated testing and coverage (during pull request)
 
 on:
   pull_request:
-    branches: [ "alpha" ]
+    branches: [ "beta", "alpha" ]
     paths:
       - 'geoprob_pipe/**'
       - 'test_system.py'

geoprob_pipe/changelog.py

@@ -1,5 +1,8 @@
 
 CHANGELOG = {
+    "2.2.12":
+        "Validatie op dubbele punten bij het inlezen van de HRD-punten. Oorzaak is database 21-2 waar een dubbeling in "
+        "zit. Direct controle op dubbeling in uittredepunten toegevoegd. ",
     "2.2.11":
         "Bug fix. When there were NaN-calculation results (PoF), it resulted in a crash in the KansElement.",
     "2.2.10":

geoprob_pipe/cmd_app/spatial_joins/coupled_hrd.py

@@ -30,8 +30,11 @@ def coupled_hrd_to_uittredepunten(app_settings: ApplicationSettings) -> bool:
         return True  # Assuming already added
 
     # Perform spatial join to find the nearest HRD-location for each Exit Point
-    gdf_exit_with_hrd = gdf_exit_points.sjoin_nearest(
+    gdf_exit_with_hrd: GeoDataFrame = gdf_exit_points.sjoin_nearest(
         gdf_hrd_locations[['geometry', 'location_name']], how='left', distance_col='distance')
+    assert gdf_exit_points.__len__() == gdf_exit_with_hrd.__len__(), \
+        (f"While coupling the HRD-location to the uittredepunten something went wrong. Please contact a developer with "
+         f"the traceback.")
 
     # Define which columns to keep (after spatial join)
     columns_to_keep = list(gdf_exit_points.columns)

geoprob_pipe/cmd_app/spatial_layers/hrd/import_from_hrd.py

@@ -6,6 +6,8 @@
 from geopandas import GeoDataFrame, read_file
 from shapely import Point
 import os
+
+from geoprob_pipe.utils.gdf import validate_unique_point_locations
 from geoprob_pipe.utils.validation_messages import BColors
 import warnings
 from geoprob_pipe.cmd_app.spatial_layers.hrd.utils import hrd_file_path
@@ -15,6 +17,7 @@
 from pandas import DataFrame, concat
 from typing import Optional, List, Tuple
 import sqlite3
+import sys
 if TYPE_CHECKING:
     from geoprob_pipe.cmd_app.cmd import ApplicationSettings
 
@@ -65,9 +68,9 @@ def _ask_path_to_hrd_dir() -> str:
     return filepath
 
 
-def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: str):
+def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: str) -> bool:
 
-    # Add HRD locations to GeoPackage
+    # Collect HRD-locations
     hrd_path = hrd_file_path(hrd_dir=hrd_dir)
     hrd = pydra.HRDatabase(hrd_path)
     location_names = hrd.locationnames
@@ -87,10 +90,33 @@ def _add_hrd_locations_to_database(app_settings: ApplicationSettings, hrd_dir: s
             "location_name": location_name,
             "geometry": Point(hrd_location.settings.x_coordinate, hrd_location.settings.y_coordinate)
         })
-
     gdf = GeoDataFrame(hrd_location_rows, columns=['location_name', 'geometry'], crs='EPSG:28992')
+    # TODO Runtime improvement: the below method of Pydra-Core, to only collect the Hydra-Locations, is relatively slow.
+    #  We could just read the sqlite HRD-database directly and store the HRD-locations.
+
+    # Validate all points are unique
+    success, failure_msg = validate_unique_point_locations(gdf=gdf, id_column="location_name")
+    if not success:
+        print(f"{BColors.WARNING}"
+              f"Er zijn één of meerdere dubbelingen in de HRD-locaties. "
+              f"Uit de validatie volgt het volgende bericht: {failure_msg} "
+              f"Corrigeer de HRD-database, of vul de overschrijdingsfrequentielijnen handmatig in. "
+              f"{BColors.ENDC}")
+
+        # Verwijder eerder gemaakte keuze 'Importeren van HRD-database' (zodat je opnieuw de keuze kunt maken)
+        file_path = app_settings.geopackage_filepath
+        conn = sqlite3.connect(file_path)
+        cursor = conn.cursor()
+        cursor.execute(f"DELETE FROM geoprob_pipe_metadata WHERE metadata_type = 'extract_hrd_data';")
+        conn.commit()
+        cursor.close()
+
+        return False
+
+    # Add HRD locations to GeoPackage
     gdf.to_file(Path(app_settings.geopackage_filepath), layer="hrd_locaties", driver="GPKG")
     print(BColors.OKBLUE, f"✅  HRD-locatie punten toegevoegd aan GeoProb-Pipe GeoPackage.", BColors.ENDC)
+    return True
 
 
 def _add_hrd_overschrijdingsfrequentielijnen(hrd_dir: str, app_settings: ApplicationSettings):
@@ -219,6 +245,11 @@ def _add_traject_parameters(app_settings: ApplicationSettings, hrd_dir: str):
 
 def import_from_hrd(app_settings: ApplicationSettings):
     hrd_dir = _ask_path_to_hrd_dir()
-    _add_hrd_locations_to_database(app_settings=app_settings, hrd_dir=hrd_dir)
+
+    # Import HRD-locations
+    if not _add_hrd_locations_to_database(app_settings=app_settings, hrd_dir=hrd_dir):
+        sys.exit("Applicatie afgesloten")
+
+    # Other actions
     _add_hrd_overschrijdingsfrequentielijnen(hrd_dir=hrd_dir, app_settings=app_settings)
     _add_traject_parameters(app_settings=app_settings, hrd_dir=hrd_dir)

geoprob_pipe/cmd_app/spatial_layers/uittredepunten/uittredepunten.py

@@ -7,6 +7,8 @@
 from shapely import Point
 from geopandas import GeoDataFrame, read_file
 import fiona
+
+from geoprob_pipe.utils.gdf import validate_unique_point_locations
 from geoprob_pipe.utils.validation_messages import BColors
 import warnings
 if TYPE_CHECKING:
@@ -189,6 +191,16 @@ def request_uittredepunten_filepath(app_settings: ApplicationSettings):
                                f"andere typen geometrie. Enkel punten zijn toegestaan.", BColors.ENDC)
         request_uittredepunten_filepath(app_settings=app_settings)
 
+    # Confirm all are unique locations
+    succes, failure_msg = validate_unique_point_locations(gdf=gdf, return_coords=True)
+    if not succes:
+        print(f"{BColors.WARNING}"
+              f"Er zijn één of meerdere dubbelingen in de uittredepunten-locaties. "
+              f"Uit de validatie volgt het volgende bericht: {failure_msg} "
+              f"Corrigeer het bestand en importeer opnieuw de uittredepunten. "
+              f"{BColors.ENDC}")
+        request_uittredepunten_filepath(app_settings=app_settings)
+
     # Continue questionnaire
     specify_column_with_maaiveld_niveau(app_settings, gdf=gdf)
 

geoprob_pipe/utils/gdf.py

@@ -55,3 +55,35 @@ def validate_vakindeling_merges_to_single_linestring(gdf: GeoDataFrame) -> Tuple
     """ Controleert of alle LineStrings aaneengesloten zijn. """
     merged = line_merge(unary_union(gdf.geometry))
     return isinstance(merged, LineString), type(merged)
+
+
+def validate_unique_point_locations(
+        gdf: GeoDataFrame, id_column: str | None = None, return_coords: bool = False) -> Tuple[bool, str]:
+    """ Validates if the given GeoDataFrame has only unique point-locations. If not, it returns a failure message
+    with in there the non-unique ids.
+
+    :param gdf:
+    :param id_column: Specifies which column identifies each point. If None, index will be used as identifiers.
+    :param return_coords: If True, the failure message will also include the RD-coordinates of the points.
+    :return:
+    """
+
+    filter_duplicate_geometries = gdf.geometry.duplicated(keep=False)
+    gdf_duplicates = gdf[filter_duplicate_geometries]
+
+    if gdf_duplicates.__len__() == 0:
+        return True, ""
+
+    # Collect ids
+    ids_of_non_unique_pnts = gdf_duplicates.index.tolist()
+    if id_column is not None:
+        ids_of_non_unique_pnts = gdf_duplicates[id_column].values.tolist()
+    failure_msg = f"Identifiers {ids_of_non_unique_pnts} have non-unique geometries."
+
+    # Collect coordinates
+    if return_coords:
+        coords = [f"{pnt.x}, {pnt.y}" for pnt in gdf_duplicates.geometry.values.tolist()]
+        id_coord_list = [f"{identifier} ({pnt})" for identifier, pnt in zip(ids_of_non_unique_pnts, coords)]
+        failure_msg = f"Identifiers {id_coord_list} have non-unique geometries."
+
+    return False, failure_msg

pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "geoprob_pipe"
-version = "2.2.11"
+version = "2.2.12"
 description = "Execute probabilistic piping calculations using the Deltares probabilic library."
 keywords = [
   "levee",