From c9e937c639fdec3b3b82e92205c266c3ad9e2908 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Tue, 26 May 2026 10:51:25 +0200 Subject: [PATCH 1/5] #348 Increase benchmark runs, lower per-iteration timeout, raise fixed-iteration ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BENCHMARK_RUNS: 1 → 30, BENCHMARK_MAX_ITERATION_SECONDS: 4500 → 3600, BENCHMARK_MAX_FIXED_WINDOW_SECONDS: 75 min → 300 min. --- src/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config.py b/src/config.py index 7d723c21..e416b1b2 100644 --- a/src/config.py +++ b/src/config.py @@ -95,7 +95,7 @@ class Config: BENCHMARK_FILE: Path = ROOT_DIR / "benchmarks.yml" RUN_ID_LENGTH: int = 6 DEFAULT_SAMPLE_TIMEOUT: float = 0.01 - BENCHMARK_RUNS: int = 1 + BENCHMARK_RUNS: int = 30 BENCHMARK_WARMUP_ITERATIONS: int = 5 BENCHMARK_ITERATIONS: int = 100 BENCHMARK_METADATA_BLOB_NAME: str = "benchmark_metadata.parquet" @@ -104,15 +104,15 @@ class Config: # Sequential stopping rule (bootstrapped CI on mean elapsed time) BENCHMARK_MIN_ITERATIONS: int = 10 BENCHMARK_MIN_TIMED_WINDOW_SECONDS: int = 60 - BENCHMARK_MAX_ITERATION_SECONDS: int = 4500 + BENCHMARK_MAX_ITERATION_SECONDS: int = 3600 BENCHMARK_MAX_TIMED_WINDOW_SECONDS: int = 6 * BENCHMARK_MAX_ITERATION_SECONDS BENCHMARK_TARGET_CI_HALF_WIDTH_RELATIVE: float = 0.05 BENCHMARK_BOOTSTRAP_RESAMPLES: int = 1000 BENCHMARK_CI_CONFIDENCE: float = 0.95 BENCHMARK_MAX_CONSECUTIVE_FAILURES: int = 3 - # Fixed-iteration cumulative wall-clock ceiling (75 min) - BENCHMARK_MAX_FIXED_WINDOW_SECONDS: int = 75 * 60 + # Fixed-iteration cumulative wall-clock ceiling (300 min) + BENCHMARK_MAX_FIXED_WINDOW_SECONDS: int = 5 * BENCHMARK_MAX_ITERATION_SECONDS INGESTION_DELAY_SECONDS: int = 600 From 0acd238b3fbe2ba41519a6eed044512b3df2e1ca Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Tue, 26 May 2026 10:51:32 +0200 Subject: [PATCH 2/5] #348 Add per-experiment runs field and skip duckdb-large RQ1 experiments: runs: 30. RQ2 experiments: runs: 3. national-scale-spatial-join-duckdb-large: skip: true (exceeds 60-min threshold). --- benchmarks.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/benchmarks.yml b/benchmarks.yml index 6b1b6c87..46c0b5b0 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -16,6 +16,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - point-in-polygon-lookup-postgis-small - point-in-polygon-lookup-local-small @@ -25,6 +26,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - point-in-polygon-lookup-duckdb-small - point-in-polygon-lookup-local-small @@ -34,6 +36,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - point-in-polygon-lookup-duckdb-small - point-in-polygon-lookup-postgis-small @@ -43,6 +46,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - point-in-polygon-lookup-postgis-large @@ -51,6 +55,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - point-in-polygon-lookup-duckdb-large @@ -60,6 +65,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - knn-search-postgis-small - knn-search-local-small @@ -69,6 +75,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - knn-search-duckdb-small - knn-search-local-small @@ -78,6 +85,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - knn-search-duckdb-small - knn-search-postgis-small @@ -87,6 +95,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - knn-search-postgis-large @@ -95,6 +104,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - knn-search-duckdb-large @@ -104,6 +114,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - bbox-filtering-postgis-small - bbox-filtering-local-small @@ -113,6 +124,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - bbox-filtering-duckdb-small - bbox-filtering-local-small @@ -122,6 +134,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - bbox-filtering-duckdb-small - bbox-filtering-postgis-small @@ -131,6 +144,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - bbox-filtering-postgis-large @@ -139,6 +153,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - bbox-filtering-duckdb-large @@ -155,6 +170,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-small - national-scale-spatial-join-databricks-partitioned-8-nodes-small @@ -165,6 +181,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-small - national-scale-spatial-join-databricks-partitioned-8-nodes-small @@ -175,6 +192,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-medium - national-scale-spatial-join-databricks-partitioned-8-nodes-medium @@ -185,6 +203,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-medium - national-scale-spatial-join-databricks-partitioned-8-nodes-medium @@ -195,6 +214,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 + skip: true # exceeds 60-min per-iteration threshold; existing data from run 2026-05-24-HBJYYT sufficient related_script_ids: - national-scale-spatial-join-postgis-large @@ -203,6 +224,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 skip: true # OOM on Azure PostgreSQL Flex Server at large scale related_script_ids: - national-scale-spatial-join-duckdb-large @@ -213,6 +235,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-medium @@ -221,6 +244,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-4-nodes-large - national-scale-spatial-join-databricks-broadcast-16-nodes-large @@ -230,6 +254,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-4-nodes-small @@ -238,6 +263,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-4-nodes-medium @@ -246,6 +272,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-2-nodes-large - national-scale-spatial-join-databricks-broadcast-16-nodes-large @@ -255,6 +282,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-8-nodes-small - national-scale-spatial-join-duckdb-small @@ -265,6 +293,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-8-nodes-medium - national-scale-spatial-join-duckdb-medium @@ -275,6 +304,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-8-nodes-large @@ -283,6 +313,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-2-nodes-large - national-scale-spatial-join-databricks-broadcast-4-nodes-large @@ -293,6 +324,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-2-nodes-medium @@ -302,6 +334,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-4-nodes-large @@ -312,6 +345,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-4-nodes-small @@ -320,6 +354,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-4-nodes-medium @@ -329,6 +364,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-large @@ -339,6 +375,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-small - national-scale-spatial-join-duckdb-small @@ -349,6 +386,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-medium @@ -360,6 +398,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-large @@ -369,6 +408,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-large From 45806062b13cbb11eaba43c5730da56e83ac608a Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Tue, 26 May 2026 10:51:39 +0200 Subject: [PATCH 3/5] #348 Skip experiments that exceed their per-experiment runs limit Orchestrator checks experiment["runs"] before launching a batch. When benchmark_run > runs, marks experiment and related peers as completed. --- main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/main.py b/main.py index d010c3f7..ba917ef4 100644 --- a/main.py +++ b/main.py @@ -74,6 +74,17 @@ def _run_benchmarks( if experiment_id in completed_experiments: continue + experiment_runs = experiment.get("runs", Config.BENCHMARK_RUNS) + if benchmark_run > experiment_runs: + completed_experiments.append(experiment_id) + for rid in experiment.get("related_script_ids", []): + completed_experiments.append(str(rid)) + logger.info( + "Skipping '%s' — benchmark run %s exceeds experiment runs limit %s.", + experiment_id, benchmark_run, experiment_runs, + ) + continue + current_batch += 1 related_experiment_ids = experiment["related_script_ids"] From 3703aa09b4d46fbb1fceab26ce7eaed4f7600f56 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Tue, 26 May 2026 11:19:23 +0200 Subject: [PATCH 4/5] #348 Replace skip: true with typed stop reasons and set logging to DEBUG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmarks.yml: skip: true → skip: timeout | skip: failed per experiment. StopReason.from_skip(): parse YAML skip value into enum. benchmark_runner: _is_skipped → _get_skip_reason returning StopReason. Config: LOGGING_LEVEL changed to DEBUG. --- benchmark_runner.py | 13 +++++++------ benchmarks.yml | 18 +++++++++--------- src/config.py | 6 ++++-- src/domain/enums/stop_reason.py | 6 ++++++ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/benchmark_runner.py b/benchmark_runner.py index 100e38b3..7d36c450 100644 --- a/benchmark_runner.py +++ b/benchmark_runner.py @@ -45,14 +45,15 @@ def benchmark_runner() -> None: run_id=run_id, benchmark_run=benchmark_run, dataset_size=dataset_size ) - if _is_skipped(script_id): - logger.info(f"Experiment '{script_id}' marked skip=true in benchmarks.yml. Recording as skipped.") + skip_reason = _get_skip_reason(script_id) + if skip_reason is not None: + logger.info(f"Experiment '{script_id}' skipped in benchmarks.yml (reason={skip_reason.value}).") _save_run_metadata( query_id=script_id, run_id=run_id, achieved_iterations=0, failed_iterations=0, - stop_reason=StopReason.FAILED, + stop_reason=skip_reason, ci_half_width_seconds=None, ci_half_width_relative=None, mean_elapsed_seconds=None, @@ -139,13 +140,13 @@ def _strip_dataset_size_suffix(script_id: str) -> str: return script_id -def _is_skipped(script_id: str) -> bool: +def _get_skip_reason(script_id: str) -> StopReason | None: with open(Config.BENCHMARK_FILE) as f: cfg = yaml.safe_load(f) for exp in cfg.get("experiments", []): if exp.get("id") == script_id and exp.get("skip"): - return True - return False + return StopReason.from_skip(exp["skip"]) + return None def _get_args() -> tuple[str, int, Optional[str], DatasetSize]: diff --git a/benchmarks.yml b/benchmarks.yml index 46c0b5b0..0ba64280 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -215,7 +215,7 @@ experiments: memory_gb: 16 dataset_size: large runs: 3 - skip: true # exceeds 60-min per-iteration threshold; existing data from run 2026-05-24-HBJYYT sufficient + skip: timeout # exceeds 60-min per-iteration threshold; existing data from run 2026-05-24-HBJYYT sufficient related_script_ids: - national-scale-spatial-join-postgis-large @@ -225,7 +225,7 @@ experiments: memory_gb: 16 dataset_size: large runs: 3 - skip: true # OOM on Azure PostgreSQL Flex Server at large scale + skip: failed # OOM on Azure PostgreSQL Flex Server at large scale related_script_ids: - national-scale-spatial-join-duckdb-large @@ -325,7 +325,7 @@ experiments: memory_gb: 16 dataset_size: medium runs: 3 - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-2-nodes-medium @@ -335,7 +335,7 @@ experiments: memory_gb: 16 dataset_size: large runs: 3 - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-4-nodes-large - national-scale-spatial-join-databricks-partitioned-16-nodes-large @@ -355,7 +355,7 @@ experiments: memory_gb: 16 dataset_size: medium runs: 3 - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-4-nodes-medium @@ -365,7 +365,7 @@ experiments: memory_gb: 16 dataset_size: large runs: 3 - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-large - national-scale-spatial-join-databricks-partitioned-16-nodes-large @@ -387,7 +387,7 @@ experiments: memory_gb: 16 dataset_size: medium runs: 3 - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-medium - national-scale-spatial-join-duckdb-medium @@ -399,7 +399,7 @@ experiments: memory_gb: 16 dataset_size: large runs: 3 - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-large @@ -409,7 +409,7 @@ experiments: memory_gb: 16 dataset_size: large runs: 3 - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-large - national-scale-spatial-join-databricks-partitioned-4-nodes-large diff --git a/src/config.py b/src/config.py index e416b1b2..319f9420 100644 --- a/src/config.py +++ b/src/config.py @@ -53,7 +53,7 @@ class Config: MVT_TILES_PATH: Path = ROOT_DIR / "resources" / "tiles.json" # LOGGING - LOGGING_LEVEL: int = logging.INFO + LOGGING_LEVEL: int = logging.DEBUG LOG_FILE: Path = LOG_DIR / f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" # GEONORGE @@ -88,7 +88,9 @@ class Config: # TEST MODE SETUP_COUNTY_LIMIT: int | None = ( - int(os.getenv("SETUP_COUNTY_LIMIT")) if os.getenv("SETUP_COUNTY_LIMIT") else None + int(os.getenv("SETUP_COUNTY_LIMIT")) + if os.getenv("SETUP_COUNTY_LIMIT") + else None ) # BENCHMARKING diff --git a/src/domain/enums/stop_reason.py b/src/domain/enums/stop_reason.py index fb0c555d..680d2adb 100644 --- a/src/domain/enums/stop_reason.py +++ b/src/domain/enums/stop_reason.py @@ -8,3 +8,9 @@ class StopReason(Enum): FIXED = "fixed" PARTIAL = "partial" FAILED = "failed" + + @classmethod + def from_skip(cls, value) -> "StopReason": + if isinstance(value, str): + return cls(value) + return cls.FAILED From 115cadac09afeaef53b80b5025c4d6e8c04c42ad Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Tue, 26 May 2026 11:22:42 +0200 Subject: [PATCH 5/5] #348 Changed logging level to INFO Signed-off-by: Jathavaan Shankarr --- src/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.py b/src/config.py index 319f9420..fede74d8 100644 --- a/src/config.py +++ b/src/config.py @@ -53,7 +53,7 @@ class Config: MVT_TILES_PATH: Path = ROOT_DIR / "resources" / "tiles.json" # LOGGING - LOGGING_LEVEL: int = logging.DEBUG + LOGGING_LEVEL: int = logging.INFO LOG_FILE: Path = LOG_DIR / f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" # GEONORGE