diff --git a/benchmark_runner.py b/benchmark_runner.py index 100e38b3..7d36c450 100644 --- a/benchmark_runner.py +++ b/benchmark_runner.py @@ -45,14 +45,15 @@ def benchmark_runner() -> None: run_id=run_id, benchmark_run=benchmark_run, dataset_size=dataset_size ) - if _is_skipped(script_id): - logger.info(f"Experiment '{script_id}' marked skip=true in benchmarks.yml. Recording as skipped.") + skip_reason = _get_skip_reason(script_id) + if skip_reason is not None: + logger.info(f"Experiment '{script_id}' skipped in benchmarks.yml (reason={skip_reason.value}).") _save_run_metadata( query_id=script_id, run_id=run_id, achieved_iterations=0, failed_iterations=0, - stop_reason=StopReason.FAILED, + stop_reason=skip_reason, ci_half_width_seconds=None, ci_half_width_relative=None, mean_elapsed_seconds=None, @@ -139,13 +140,13 @@ def _strip_dataset_size_suffix(script_id: str) -> str: return script_id -def _is_skipped(script_id: str) -> bool: +def _get_skip_reason(script_id: str) -> StopReason | None: with open(Config.BENCHMARK_FILE) as f: cfg = yaml.safe_load(f) for exp in cfg.get("experiments", []): if exp.get("id") == script_id and exp.get("skip"): - return True - return False + return StopReason.from_skip(exp["skip"]) + return None def _get_args() -> tuple[str, int, Optional[str], DatasetSize]: diff --git a/benchmarks.yml b/benchmarks.yml index 6b1b6c87..0ba64280 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -16,6 +16,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - point-in-polygon-lookup-postgis-small - point-in-polygon-lookup-local-small @@ -25,6 +26,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - point-in-polygon-lookup-duckdb-small - point-in-polygon-lookup-local-small @@ -34,6 +36,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - point-in-polygon-lookup-duckdb-small - point-in-polygon-lookup-postgis-small @@ -43,6 +46,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - point-in-polygon-lookup-postgis-large @@ -51,6 +55,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - point-in-polygon-lookup-duckdb-large @@ -60,6 +65,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - knn-search-postgis-small - knn-search-local-small @@ -69,6 +75,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - knn-search-duckdb-small - knn-search-local-small @@ -78,6 +85,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - knn-search-duckdb-small - knn-search-postgis-small @@ -87,6 +95,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - knn-search-postgis-large @@ -95,6 +104,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - knn-search-duckdb-large @@ -104,6 +114,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - bbox-filtering-postgis-small - bbox-filtering-local-small @@ -113,6 +124,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - bbox-filtering-duckdb-small - bbox-filtering-local-small @@ -122,6 +134,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 30 related_script_ids: - bbox-filtering-duckdb-small - bbox-filtering-postgis-small @@ -131,6 +144,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - bbox-filtering-postgis-large @@ -139,6 +153,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 30 related_script_ids: - bbox-filtering-duckdb-large @@ -155,6 +170,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-small - national-scale-spatial-join-databricks-partitioned-8-nodes-small @@ -165,6 +181,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-small - national-scale-spatial-join-databricks-partitioned-8-nodes-small @@ -175,6 +192,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-medium - national-scale-spatial-join-databricks-partitioned-8-nodes-medium @@ -185,6 +203,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-medium - national-scale-spatial-join-databricks-partitioned-8-nodes-medium @@ -195,6 +214,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 + skip: timeout # exceeds 60-min per-iteration threshold; existing data from run 2026-05-24-HBJYYT sufficient related_script_ids: - national-scale-spatial-join-postgis-large @@ -203,7 +224,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large - skip: true # OOM on Azure PostgreSQL Flex Server at large scale + runs: 3 + skip: failed # OOM on Azure PostgreSQL Flex Server at large scale related_script_ids: - national-scale-spatial-join-duckdb-large @@ -213,6 +235,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-medium @@ -221,6 +244,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-4-nodes-large - national-scale-spatial-join-databricks-broadcast-16-nodes-large @@ -230,6 +254,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-4-nodes-small @@ -238,6 +263,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-4-nodes-medium @@ -246,6 +272,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-2-nodes-large - national-scale-spatial-join-databricks-broadcast-16-nodes-large @@ -255,6 +282,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-8-nodes-small - national-scale-spatial-join-duckdb-small @@ -265,6 +293,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-8-nodes-medium - national-scale-spatial-join-duckdb-medium @@ -275,6 +304,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-partitioned-8-nodes-large @@ -283,6 +313,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-2-nodes-large - national-scale-spatial-join-databricks-broadcast-4-nodes-large @@ -293,7 +324,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale + runs: 3 + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-2-nodes-medium @@ -302,7 +334,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + runs: 3 + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-4-nodes-large - national-scale-spatial-join-databricks-partitioned-16-nodes-large @@ -312,6 +345,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-4-nodes-small @@ -320,7 +354,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale + runs: 3 + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-4-nodes-medium @@ -329,7 +364,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + runs: 3 + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-large - national-scale-spatial-join-databricks-partitioned-16-nodes-large @@ -339,6 +375,7 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: small + runs: 3 related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-small - national-scale-spatial-join-duckdb-small @@ -349,7 +386,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: medium - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale + runs: 3 + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at medium scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-medium - national-scale-spatial-join-duckdb-medium @@ -360,7 +398,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + runs: 3 + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-broadcast-8-nodes-large @@ -369,7 +408,8 @@ experiments: cpu: 4 memory_gb: 16 dataset_size: large - skip: true # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale + runs: 3 + skip: failed # executor OOM: RangeJoin spatial index exceeds Standard_D4s_v3 memory at large scale related_script_ids: - national-scale-spatial-join-databricks-partitioned-2-nodes-large - national-scale-spatial-join-databricks-partitioned-4-nodes-large diff --git a/main.py b/main.py index d010c3f7..ba917ef4 100644 --- a/main.py +++ b/main.py @@ -74,6 +74,17 @@ def _run_benchmarks( if experiment_id in completed_experiments: continue + experiment_runs = experiment.get("runs", Config.BENCHMARK_RUNS) + if benchmark_run > experiment_runs: + completed_experiments.append(experiment_id) + for rid in experiment.get("related_script_ids", []): + completed_experiments.append(str(rid)) + logger.info( + "Skipping '%s' — benchmark run %s exceeds experiment runs limit %s.", + experiment_id, benchmark_run, experiment_runs, + ) + continue + current_batch += 1 related_experiment_ids = experiment["related_script_ids"] diff --git a/src/config.py b/src/config.py index 7d723c21..fede74d8 100644 --- a/src/config.py +++ b/src/config.py @@ -88,14 +88,16 @@ class Config: # TEST MODE SETUP_COUNTY_LIMIT: int | None = ( - int(os.getenv("SETUP_COUNTY_LIMIT")) if os.getenv("SETUP_COUNTY_LIMIT") else None + int(os.getenv("SETUP_COUNTY_LIMIT")) + if os.getenv("SETUP_COUNTY_LIMIT") + else None ) # BENCHMARKING BENCHMARK_FILE: Path = ROOT_DIR / "benchmarks.yml" RUN_ID_LENGTH: int = 6 DEFAULT_SAMPLE_TIMEOUT: float = 0.01 - BENCHMARK_RUNS: int = 1 + BENCHMARK_RUNS: int = 30 BENCHMARK_WARMUP_ITERATIONS: int = 5 BENCHMARK_ITERATIONS: int = 100 BENCHMARK_METADATA_BLOB_NAME: str = "benchmark_metadata.parquet" @@ -104,15 +106,15 @@ class Config: # Sequential stopping rule (bootstrapped CI on mean elapsed time) BENCHMARK_MIN_ITERATIONS: int = 10 BENCHMARK_MIN_TIMED_WINDOW_SECONDS: int = 60 - BENCHMARK_MAX_ITERATION_SECONDS: int = 4500 + BENCHMARK_MAX_ITERATION_SECONDS: int = 3600 BENCHMARK_MAX_TIMED_WINDOW_SECONDS: int = 6 * BENCHMARK_MAX_ITERATION_SECONDS BENCHMARK_TARGET_CI_HALF_WIDTH_RELATIVE: float = 0.05 BENCHMARK_BOOTSTRAP_RESAMPLES: int = 1000 BENCHMARK_CI_CONFIDENCE: float = 0.95 BENCHMARK_MAX_CONSECUTIVE_FAILURES: int = 3 - # Fixed-iteration cumulative wall-clock ceiling (75 min) - BENCHMARK_MAX_FIXED_WINDOW_SECONDS: int = 75 * 60 + # Fixed-iteration cumulative wall-clock ceiling (300 min) + BENCHMARK_MAX_FIXED_WINDOW_SECONDS: int = 5 * BENCHMARK_MAX_ITERATION_SECONDS INGESTION_DELAY_SECONDS: int = 600 diff --git a/src/domain/enums/stop_reason.py b/src/domain/enums/stop_reason.py index fb0c555d..680d2adb 100644 --- a/src/domain/enums/stop_reason.py +++ b/src/domain/enums/stop_reason.py @@ -8,3 +8,9 @@ class StopReason(Enum): FIXED = "fixed" PARTIAL = "partial" FAILED = "failed" + + @classmethod + def from_skip(cls, value) -> "StopReason": + if isinstance(value, str): + return cls(value) + return cls.FAILED