From ad32874bc6fd2b33e591ec4ba4af6a07f9be3409 Mon Sep 17 00:00:00 2001 From: amarcozzi Date: Mon, 15 Jun 2026 22:52:06 -0600 Subject: [PATCH] Replace inventory in-place full rewrite with partial partition rewrite In-place modifications and treatments rewrote the entire Parquet dataset via a delete + copy-back staging swap, regardless of how many rows the operation actually touched. Rewrite only the partitions whose content changes. standgen storage: - write_changed_partitions(id, transform): read every partition in one concurrent gcsfs cat, apply the per-partition transform, and write back only the partitions where not new.equals(old) in one concurrent pipe. Leaves _metadata untouched (the file set is unchanged; readers re-read each footer). Replaces save_parquet_replace. - write_full_partitions(id, df): materialized full rewrite for the one case that re-partitions globally; drops the now-wrong _metadata / _common_metadata and stale part files so readers list fresh. Modifications route every mod through write_changed_partitions. Treatments route by per-partition expressibility, not metric: diameter and directional basal-area thins (the latter reduced to a single precomputed diameter cutoff that reproduces the fastfuels-core thinner exactly) rewrite only changed partitions, including polygon-scoped thins; only proportional basal-area (random whole-stand removal) falls back to write_full_partitions. API metadata/data endpoints read num_partitions / total_rows / columns from the dask DataFrame (per-file footers) instead of the aggregated _metadata, so counts stay correct after a partial rewrite leaves _metadata stale. Drops the per-partition row-count list from the /data metadata response. Adds an end-to-end benchmark (benchmarks/bench_inplace_inventory.py) that times in-place modifications against the deployed API + standgen pipeline. Closes #355 --- .../api/api/resources/inventories/cache.py | 73 ++- .../api/api/resources/inventories/router.py | 18 +- .../api/api/resources/inventories/schema.py | 6 - services/api/benchmarks/.gitignore | 2 + .../api/benchmarks/bench_inplace_inventory.py | 428 ++++++++++++++++++ .../resources/inventories/test_data_cache.py | 74 ++- .../resources/inventories/test_data_router.py | 6 +- .../resources/inventories/test_schema.py | 19 - .../standgen/handlers/modifications.py | 19 +- .../standgen/standgen/handlers/treatments.py | 51 ++- services/standgen/standgen/storage.py | 140 +++--- services/standgen/standgen/treatments.py | 129 ++++++ .../tests/handlers/test_modifications.py | 101 ++--- .../tests/handlers/test_treatments.py | 219 +++++---- services/standgen/tests/test_storage.py | 101 ++++- 15 files changed, 1062 insertions(+), 324 deletions(-) create mode 100644 services/api/benchmarks/.gitignore create mode 100644 services/api/benchmarks/bench_inplace_inventory.py diff --git a/services/api/api/resources/inventories/cache.py b/services/api/api/resources/inventories/cache.py index 78c9db70..7cc6433a 100644 --- a/services/api/api/resources/inventories/cache.py +++ b/services/api/api/resources/inventories/cache.py @@ -1,60 +1,41 @@ """ api/v2/resources/inventories/cache.py -Cached async parquet metadata access for inventory data streaming. +Cached async access to an inventory's partitioned-Parquet metadata and data. + +Counts come from the dask DataFrame, which reads each partition's own footer (and +ignores any aggregated ``_metadata``). This stays correct after standgen rewrites +only the partitions a modification changed — an in-place overwrite leaves the +``_metadata`` aggregate stale, but ``len(ddf)`` does not trust it. """ import asyncio from dataclasses import dataclass +import dask.dataframe as dd import pandas as pd -import pyarrow.parquet as pq from ring import lru from lib.config import INVENTORIES_BUCKET -@dataclass -class PartitionMeta: - index: int - num_rows: int - path: str - - @dataclass class InventoryMeta: num_partitions: int total_rows: int columns: list[str] - partitions: list[PartitionMeta] - - -def _read_metadata_sync(inventory_id: str) -> InventoryMeta: - metadata_path = f"gs://{INVENTORIES_BUCKET}/{inventory_id}/_metadata" - pf = pq.ParquetFile(metadata_path) - return _parse_metadata(pf.metadata) -def _parse_metadata(metadata: pq.FileMetaData) -> InventoryMeta: - partitions_by_path: dict[str, int] = {} - for i in range(metadata.num_row_groups): - rg = metadata.row_group(i) - path = rg.column(0).file_path - partitions_by_path[path] = partitions_by_path.get(path, 0) + rg.num_rows +def _inventory_path(inventory_id: str) -> str: + return f"gs://{INVENTORIES_BUCKET}/{inventory_id}" - columns = metadata.schema.to_arrow_schema().names - - sorted_paths = sorted(partitions_by_path.keys()) - partitions = [ - PartitionMeta(index=idx, num_rows=partitions_by_path[path], path=path) - for idx, path in enumerate(sorted_paths) - ] +def _read_metadata_sync(inventory_id: str) -> InventoryMeta: + ddf = dd.read_parquet(_inventory_path(inventory_id)) return InventoryMeta( - num_partitions=len(partitions), - total_rows=sum(p.num_rows for p in partitions), - columns=columns, - partitions=partitions, + num_partitions=ddf.npartitions, + total_rows=len(ddf), # footer-level row count; ignores stale _metadata + columns=list(ddf.columns), ) @@ -62,23 +43,29 @@ def _parse_metadata(metadata: pq.FileMetaData) -> InventoryMeta: async def get_inventory_metadata( inventory_id: str, checksum: str | None ) -> InventoryMeta: - """Read an inventory's partitioned-Parquet metadata, cached per content. + """Read an inventory's partition metadata, cached per content. ``checksum`` participates only in the cache key (the body ignores it). An - in-place modification (POST .../modifications) rewrites the data under the - same ``inventory_id`` and re-assigns the inventory's ``checksum``, so a new - checksum bypasses the stale entry. Without it, this LRU would keep serving - pre-modification partition paths and row counts. ``None`` (legacy - inventories without a checksum) is a valid, stable key — such inventories - predate in-place edits, and any in-place edit assigns a fresh checksum. + in-place modification rewrites the changed partitions under the same + ``inventory_id`` and re-assigns the inventory's ``checksum``, so a new + checksum bypasses the stale entry. ``None`` (legacy inventories without a + checksum) is a valid, stable key. """ return await asyncio.to_thread(_read_metadata_sync, inventory_id) +def _read_partition_sync( + inventory_id: str, partition_index: int, columns: list[str] | None +) -> pd.DataFrame: + ddf = dd.read_parquet(_inventory_path(inventory_id), columns=columns) + return ddf.partitions[partition_index].compute().reset_index(drop=True) + + async def read_partition( inventory_id: str, - partition_path: str, + partition_index: int, columns: list[str] | None = None, ) -> pd.DataFrame: - path = f"gs://{INVENTORIES_BUCKET}/{inventory_id}/{partition_path}" - return await asyncio.to_thread(pd.read_parquet, path, columns=columns) + return await asyncio.to_thread( + _read_partition_sync, inventory_id, partition_index, columns + ) diff --git a/services/api/api/resources/inventories/router.py b/services/api/api/resources/inventories/router.py index cbf3a641..e8332296 100644 --- a/services/api/api/resources/inventories/router.py +++ b/services/api/api/resources/inventories/router.py @@ -45,7 +45,6 @@ InventoryDataMetadata, InventoryDataResponse, InventoryJsonOrientation, - InventoryPartitionInfo, InventorySortField, InventoryType, ListInventoriesResponse, @@ -599,9 +598,9 @@ async def get_inventory_data_metadata( """ # Get Inventory Data Metadata - Returns partition count, total rows, per-partition row counts, and column - names for a completed inventory. Reads only the `_metadata` file from GCS - (cached after first access). + Returns the partition count, total row count, and column names for a + completed inventory (cached after first access). To get a single partition's + rows — and its row count — fetch it from the partition data endpoint below. ## Path Parameters @@ -611,10 +610,10 @@ async def get_inventory_data_metadata( ## Response - **inventory_id**: The inventory ID. - - **num_partitions**: Number of Parquet partitions. + - **num_partitions**: Number of Parquet partitions (addressable as `0` to + `num_partitions - 1` via the partition data endpoint). - **total_rows**: Total row count across all partitions. - **columns**: List of column names. - - **partitions**: Per-partition index and row count. ## Error Responses @@ -647,10 +646,6 @@ async def get_inventory_data_metadata( num_partitions=meta.num_partitions, total_rows=meta.total_rows, columns=meta.columns, - partitions=[ - InventoryPartitionInfo(index=p.index, num_rows=p.num_rows) - for p in meta.partitions - ], ) @@ -748,8 +743,7 @@ async def get_inventory_data( detail=(f"Columns not found: {missing}. Available: {meta.columns}"), ) - partition = meta.partitions[partition_index] - df = await read_partition(inventory_id, partition.path, columns=requested_columns) + df = await read_partition(inventory_id, partition_index, columns=requested_columns) col_names = list(df.columns) diff --git a/services/api/api/resources/inventories/schema.py b/services/api/api/resources/inventories/schema.py index 42c1467e..dc58d6a3 100644 --- a/services/api/api/resources/inventories/schema.py +++ b/services/api/api/resources/inventories/schema.py @@ -211,17 +211,11 @@ class InventoryJsonOrientation(StrEnum): records = "records" -class InventoryPartitionInfo(BaseModel): - index: int - num_rows: int - - class InventoryDataMetadata(BaseModel): inventory_id: str num_partitions: int total_rows: int columns: list[str] - partitions: list[InventoryPartitionInfo] class InventoryDataResponse(BaseModel): diff --git a/services/api/benchmarks/.gitignore b/services/api/benchmarks/.gitignore new file mode 100644 index 00000000..e6d76c8f --- /dev/null +++ b/services/api/benchmarks/.gitignore @@ -0,0 +1,2 @@ +# Benchmark result files (timestamped, carry run-specific resource IDs). +bench_inplace_*.json diff --git a/services/api/benchmarks/bench_inplace_inventory.py b/services/api/benchmarks/bench_inplace_inventory.py new file mode 100644 index 00000000..ada75523 --- /dev/null +++ b/services/api/benchmarks/bench_inplace_inventory.py @@ -0,0 +1,428 @@ +""" +End-to-end timing benchmark for in-place inventory modifications. + +Drives the *deployed* API + standgen pipeline (not a unit/integration test) to +measure how long an in-place ``POST .../{inventory_id}/modifications`` takes to +complete, end to end. Used to A/B the storage rewrite in #355: run once against +the currently-deployed full-rewrite standgen, then again after deploying the +partial-rewrite update, and compare. + +What it does: + 1. Creates a sized domain, a TreeMap PIM grid, and a PIM tree inventory + (deterministic by ``--seed`` so before/after runs operate on identical data). + 2. Reads the inventory's partition count / row count for context. + 3. Times two modification scenarios, each repeated ``--reps`` times + (the first rep per scenario is discarded as a cold-start warmup): + - ``scoped``: multiply tree height inside a sub-region of the domain. + Touches only the partitions that region overlaps -> the partial-rewrite + win shows up here. + - ``global``: multiply every tree's height. Touches all partitions -> + a no-regression check (partial == full when everything changes). + A multiply (not a remove) is used so every rep does the same work on a + stable partition layout (no rows are dropped between reps). + 4. Writes results to ``bench_inplace_