PiP fixed probe + polygon return, suppress KNN warning, remove 12-node configs

src/config.py

@@ -70,9 +70,8 @@ class Config:
     )
     TRONDHEIM_CENTER_WGS84: tuple[float, float] = (10.395, 63.430)
     KNN_SEARCH_K: int = 10
-    POINT_IN_POLYGON_TOTAL_POINTS: int = 10
-    POINT_IN_POLYGON_INSIDE_RATIO: float = 0.3
-    POINT_IN_POLYGON_PROBE_SEED: int = 42
+    # NTNU Hovedbygget (Main Building) — large footprint guaranteed in the buildings dataset
+    POINT_IN_POLYGON_PROBE_WGS84: tuple[float, float] = (10.4044, 63.4187)
     VECTOR_TILES_100K_TOTAL_REQUESTS: int = 100_000
 
     # STAC

src/presentation/configuration/app_config.py

@@ -46,12 +46,10 @@ def initialize_dependencies(
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_broadcast_2_nodes",
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_broadcast_4_nodes",
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_broadcast_8_nodes",
-            "src.presentation.entrypoints.national_scale_spatial_join_databricks_broadcast_12_nodes",
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_broadcast_16_nodes",
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_2_nodes",
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_4_nodes",
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_8_nodes",
-            "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_12_nodes",
             "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_16_nodes",
             "src.presentation.entrypoints.setup_benchmarking_framework",
 

src/presentation/entrypoints/knn_search_local.py

@@ -1,3 +1,5 @@
+import warnings
+
 import geopandas as gpd
 from shapely.geometry import Point
 
@@ -46,7 +48,13 @@ def _benchmark() -> gpd.GeoDataFrame:
         gdf = gpd.read_file(Config.BUILDINGS_SHAPEFILE)
         gdf = gdf.set_crs(epsg=4326, allow_override=True)
 
-        distances = gdf.geometry.distance(reference)
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="Geometry is in a geographic CRS",
+                category=UserWarning,
+            )
+            distances = gdf.geometry.distance(reference)
         nearest_idx = distances.nsmallest(Config.KNN_SEARCH_K).index
         return gdf.loc[nearest_idx]
 

src/presentation/entrypoints/point_in_polygon_lookup_duckdb.py

@@ -1,13 +1,11 @@
-import random
-
 from dependency_injector.wiring import Provide, inject
 from duckdb import DuckDBPyConnection
 
 from src import Config
 from src.application.common.monitor import monitor
 from src.application.contracts import IFilePathService
 from src.application.dtos import CostConfiguration
-from src.domain.enums import StorageContainer, Theme, BenchmarkIteration, BoundingBox, DatasetSize
+from src.domain.enums import StorageContainer, Theme, BenchmarkIteration, DatasetSize
 from src.infra.infrastructure import Containers
 from src.presentation.entrypoints._factory import _build_query_id, _get_dataset_size
 
@@ -18,74 +16,13 @@ def point_in_polygon_lookup_duckdb(
     path_service: IFilePathService = Provide[Containers.file_path_service],
 ) -> None:
     """
-    Benchmark: point-in-polygon lookups against the buildings dataset using DuckDB's
-    spatial extension over Azure Blob Storage. The dataset size is pulled from DI and
-    parameterises the parquet path. Generates a mix of probe points guaranteed to
-    fall inside buildings and uniformly random points within the Trondheim bounding
-    box (which may or may not land on a building) up front, then times per-point
-    ``ST_Contains`` counts.
+    Benchmark: point-in-polygon lookup against the buildings dataset using DuckDB's
+    spatial extension over Azure Blob Storage. Tests a single fixed query point
+    against all building polygons and returns the containing polygon(s).
     """
     dataset_size = _get_dataset_size()
-    points = _generate_points(
-        db_context=db_context, path_service=path_service, dataset_size=dataset_size
-    )
     benchmark_fn = _build_benchmark_fn(dataset_size=dataset_size)
-    benchmark_fn(points=points, db_context=db_context, path_service=path_service)
-
-
-def _generate_points(
-    db_context: DuckDBPyConnection,
-    path_service: IFilePathService,
-    dataset_size: DatasetSize,
-) -> list[tuple[float, float]]:
-    min_lon, min_lat, max_lon, max_lat = BoundingBox.TRONDHEIM_WGS84.value
-    n_inside = int(Config.POINT_IN_POLYGON_TOTAL_POINTS * Config.POINT_IN_POLYGON_INSIDE_RATIO)
-    n_random = Config.POINT_IN_POLYGON_TOTAL_POINTS - n_inside
-
-    path = path_service.create_release_virtual_filesystem_path(
-        storage_scheme="az",
-        release=Config.BENCHMARK_DOPPA_DATA_RELEASE,
-        container=StorageContainer.DATA,
-        theme=Theme.BUILDINGS,
-        dataset_size=dataset_size,
-        region="*",
-        file_name="*.parquet",
-    )
-
-    # TODO: See if this query can be improved in terms of efficiency
-    rows = db_context.execute(
-        f"""
-        WITH buildings_with_point_on_surface AS (
-            SELECT *, ST_PointOnSurface(geometry) AS point_on_surface FROM read_parquet('{path}')
-        ),
-
-        buildings AS(
-            SELECT
-                ST_X(bpof.point_on_surface) AS lon,
-                ST_Y(bpof.point_on_surface) AS lat
-            FROM buildings_with_point_on_surface bpof
-            WHERE ST_Intersects(geometry, ST_MakeEnvelope(?, ?, ?, ?)) AND ST_IsValid(geometry)
-            ORDER BY lon, lat
-            LIMIT ?
-        )
-
-        SELECT * FROM buildings;
-        """,
-        [min_lon, min_lat, max_lon, max_lat, n_inside],
-    ).fetchall()
-
-    inside_points = [(row[0], row[1]) for row in rows]
-
-    # TODO: Explore comments from https://github.com/kartAI/doppa/pull/196
-    rng = random.Random(Config.POINT_IN_POLYGON_PROBE_SEED)
-    random_bbox_points = [
-        (rng.uniform(min_lon, max_lon), rng.uniform(min_lat, max_lat))
-        for _ in range(n_random)
-    ]
-
-    combined = inside_points + random_bbox_points
-    rng.shuffle(combined)
-    return combined
+    benchmark_fn(db_context=db_context, path_service=path_service)
 
 
 def _build_benchmark_fn(dataset_size: DatasetSize):
@@ -97,7 +34,6 @@ def _build_benchmark_fn(dataset_size: DatasetSize):
         cost_configuration=CostConfiguration(include_aci=True, include_blob_storage=True),
     )
     def _benchmark(
-        points: list[tuple[float, float]],
         db_context: DuckDBPyConnection,
         path_service: IFilePathService,
     ) -> list:
@@ -111,17 +47,13 @@ def _benchmark(
             file_name="*.parquet",
         )
 
-        rows: list = []
-        for lon, lat in points:
-            rows.extend(
-                db_context.execute(
-                    f"""
-                    SELECT COUNT(*) FROM read_parquet('{path}')
-                    WHERE ST_Contains(geometry, ST_Point(?, ?))
-                    """,
-                    [lon, lat],
-                ).fetchall()
-            )
-        return rows
+        lon, lat = Config.POINT_IN_POLYGON_PROBE_WGS84
+        return db_context.execute(
+            f"""
+            SELECT * FROM read_parquet('{path}')
+            WHERE ST_Contains(geometry, ST_Point(?, ?))
+            """,
+            [lon, lat],
+        ).fetchall()
 
     return _benchmark

src/presentation/entrypoints/point_in_polygon_lookup_local.py

@@ -1,26 +1,20 @@
-import random
-
 import geopandas as gpd
-from shapely.geometry import Point, box
+from shapely.geometry import Point
 
 from src import Config
 from src.application.common.monitor import monitor
 from src.application.dtos import CostConfiguration
-from src.domain.enums import BenchmarkIteration, BoundingBox, DatasetSize
+from src.domain.enums import BenchmarkIteration, DatasetSize
 from src.presentation.entrypoints._factory import _build_query_id, _get_dataset_size
 from src.presentation.entrypoints._shapefile import download_buildings_shapefile
 
 
 def point_in_polygon_lookup_local() -> None:
     """
-    Benchmark: point-in-polygon lookups against the buildings shapefile, executed
-    locally via GeoPandas. The Shapefile target is the small dataset only per
-    Table 4.2.1, so this entrypoint refuses to run at MEDIUM or LARGE rather than
-    silently producing comparable numbers. Downloads the pre-baked shapefile copy
-    from blob storage, generates a mix of probe points guaranteed to fall inside
-    buildings and uniformly random points within the Trondheim bounding box (which
-    may or may not land on a building) up front, then times per-point
-    ``contains`` counts.
+    Benchmark: point-in-polygon lookup against the buildings shapefile, executed
+    locally via GeoPandas. Tests a single fixed query point against all building
+    polygons and returns the containing polygon(s). The Shapefile target is the
+    small dataset only per Table 4.2.1.
     """
     dataset_size = _get_dataset_size()
     if dataset_size is not DatasetSize.SMALL:
@@ -30,33 +24,8 @@ def point_in_polygon_lookup_local() -> None:
         )
     download_buildings_shapefile()
     gdf = gpd.read_file(Config.BUILDINGS_SHAPEFILE).set_crs(epsg=4326, allow_override=True)
-    points = _generate_points(gdf=gdf)
     benchmark_fn = _build_benchmark_fn(dataset_size=dataset_size, gdf=gdf)
-    benchmark_fn(points=points)
-
-
-def _generate_points(gdf: gpd.GeoDataFrame) -> list[tuple[float, float]]:
-    min_lon, min_lat, max_lon, max_lat = BoundingBox.TRONDHEIM_WGS84.value
-    n_inside = int(Config.POINT_IN_POLYGON_TOTAL_POINTS * Config.POINT_IN_POLYGON_INSIDE_RATIO)
-    n_random = Config.POINT_IN_POLYGON_TOTAL_POINTS - n_inside
-
-    envelope = box(min_lon, min_lat, max_lon, max_lat)
-    inside_buildings = gdf[gdf.geometry.is_valid & gdf.geometry.intersects(envelope)]
-    inside_reps = inside_buildings.geometry.representative_point()
-    inside_sorted = sorted(
-        ((pt.x, pt.y) for pt in inside_reps), key=lambda p: (p[0], p[1])
-    )
-    inside_points = inside_sorted[:n_inside]
-
-    rng = random.Random(Config.POINT_IN_POLYGON_PROBE_SEED)
-    random_bbox_points = [
-        (rng.uniform(min_lon, max_lon), rng.uniform(min_lat, max_lat))
-        for _ in range(n_random)
-    ]
-
-    combined = list(inside_points) + random_bbox_points
-    rng.shuffle(combined)
-    return combined
+    benchmark_fn()
 
 
 def _build_benchmark_fn(dataset_size: DatasetSize, gdf: gpd.GeoDataFrame):
@@ -67,10 +36,9 @@ def _build_benchmark_fn(dataset_size: DatasetSize, gdf: gpd.GeoDataFrame):
         benchmark_iteration=BenchmarkIteration.POINT_IN_POLYGON_LOOKUP,
         cost_configuration=CostConfiguration(include_aci=True, include_blob_storage=False),
     )
-    def _benchmark(points: list[tuple[float, float]]) -> list[int]:
-        results: list[int] = []
-        for lon, lat in points:
-            results.append(int(gdf.geometry.contains(Point(lon, lat)).sum()))
-        return results
+    def _benchmark() -> gpd.GeoDataFrame:
+        lon, lat = Config.POINT_IN_POLYGON_PROBE_WGS84
+        point = Point(lon, lat)
+        return gdf[gdf.geometry.contains(point)]
 
     return _benchmark

src/presentation/entrypoints/point_in_polygon_lookup_postgis.py

@@ -1,12 +1,10 @@
-import random
-
 from dependency_injector.wiring import Provide, inject
 from sqlalchemy import Engine, text
 
 from src import Config
 from src.application.common.monitor import monitor
 from src.application.dtos import CostConfiguration
-from src.domain.enums import BenchmarkIteration, BoundingBox, DatasetSize
+from src.domain.enums import BenchmarkIteration, DatasetSize
 from src.infra.infrastructure import Containers
 from src.presentation.entrypoints._factory import _build_query_id, _get_dataset_size
 
@@ -16,71 +14,13 @@ def point_in_polygon_lookup_postgis(
     db_context: Engine = Provide[Containers.postgres_context],
 ) -> None:
     """
-    Benchmark: point-in-polygon lookups against the seeded ``buildings_{size}`` table
-    using PostGIS. The dataset size is pulled from DI and parameterises the buildings
-    table reference. Generates a mix of probe points guaranteed to fall inside
-    buildings and uniformly random points within the Trondheim bounding box (which
-    may or may not land on a building) up front, then times per-point
-    ``ST_Contains`` counts.
+    Benchmark: point-in-polygon lookup against the seeded ``buildings_{size}`` table
+    using PostGIS. Tests a single fixed query point against all building polygons
+    and returns the containing polygon(s).
     """
     dataset_size = _get_dataset_size()
-    points = _generate_points(db_context=db_context, dataset_size=dataset_size)
     benchmark_fn = _build_benchmark_fn(dataset_size=dataset_size)
-    benchmark_fn(points=points, db_context=db_context)
-
-
-def _generate_points(
-    db_context: Engine, dataset_size: DatasetSize
-) -> list[tuple[float, float]]:
-    min_lon, min_lat, max_lon, max_lat = BoundingBox.TRONDHEIM_WGS84.value
-    n_inside = int(Config.POINT_IN_POLYGON_TOTAL_POINTS * Config.POINT_IN_POLYGON_INSIDE_RATIO)
-    n_random = Config.POINT_IN_POLYGON_TOTAL_POINTS - n_inside
-
-    buildings_table = f"buildings_{dataset_size.value}"
-
-    # TODO: See if this query can be improved in terms of efficiency
-    sql = text(f"""
-        WITH buildings_with_point_on_surface AS (
-            SELECT *, ST_PointOnSurface(geometry) AS point_on_surface FROM {buildings_table}
-        ),
-
-        buildings_inside AS(
-            SELECT
-                ST_X(bpof.point_on_surface) AS lon,
-                ST_Y(bpof.point_on_surface) AS lat
-            FROM buildings_with_point_on_surface bpof
-            WHERE ST_Intersects(geometry, ST_MakeEnvelope(:min_lon, :min_lat, :max_lon, :max_lat, 4326)) AND ST_IsValid(geometry)
-            ORDER BY lon, lat
-            LIMIT :limit
-        )
-
-        SELECT * FROM buildings_inside;
-        """)
-
-    with db_context.connect() as conn:
-        rows = conn.execute(
-            sql,
-            {
-                "min_lon": min_lon,
-                "min_lat": min_lat,
-                "max_lon": max_lon,
-                "max_lat": max_lat,
-                "limit": n_inside,
-            },
-        ).fetchall()
-
-    inside_points = [(row[0], row[1]) for row in rows]
-
-    # TODO: Explore comments from https://github.com/kartAI/doppa/pull/196
-    rng = random.Random(Config.POINT_IN_POLYGON_PROBE_SEED)
-    random_bbox_points = [
-        (rng.uniform(min_lon, max_lon), rng.uniform(min_lat, max_lat))
-        for _ in range(n_random)
-    ]
-
-    combined = inside_points + random_bbox_points
-    rng.shuffle(combined)
-    return combined
+    benchmark_fn(db_context=db_context)
 
 
 def _build_benchmark_fn(dataset_size: DatasetSize):
@@ -93,19 +33,16 @@ def _build_benchmark_fn(dataset_size: DatasetSize):
         cost_configuration=CostConfiguration(include_aci=True, include_postgres=True),
     )
     def _benchmark(
-        points: list[tuple[float, float]],
         db_context: Engine,
     ) -> list:
         sql = text(f"""
-            SELECT COUNT(*)
+            SELECT *
             FROM {buildings_table}
             WHERE ST_Contains(geometry, ST_SetSRID(ST_Point(:lon, :lat), 4326))
             """)
 
-        results: list = []
+        lon, lat = Config.POINT_IN_POLYGON_PROBE_WGS84
         with db_context.connect() as conn:
-            for lon, lat in points:
-                results.append(conn.execute(sql, {"lon": lon, "lat": lat}).scalar_one())
-        return results
+            return conn.execute(sql, {"lon": lon, "lat": lat}).fetchall()
 
     return _benchmark